aboutsummaryrefslogtreecommitdiffstats
path: root/storage/src/tests/distributor
diff options
context:
space:
mode:
authorTor Brede Vekterli <vekterli@verizonmedia.com>2021-09-20 14:24:41 +0200
committerGitHub <noreply@github.com>2021-09-20 14:24:41 +0200
commita26d83aab1acbe84ce31a7f886374e33202ec0d0 (patch)
treea09b952441bc566393b65ffc58a18d475d806f41 /storage/src/tests/distributor
parent23a050f3ae7755670a1b13e9f26fe8b19fb81042 (diff)
parent830bd44f5fa5cb5a0a2ee7ca34e8afaff40c474d (diff)
Merge pull request #19202 from vespa-engine/vekterli/remove-most-of-distributor-legacy-mode
Remove most traces of distributor legacy mode [run-systemtest]
Diffstat (limited to 'storage/src/tests/distributor')
-rw-r--r--storage/src/tests/distributor/CMakeLists.txt3
-rw-r--r--storage/src/tests/distributor/distributor_stripe_test_util.cpp5
-rw-r--r--storage/src/tests/distributor/distributor_stripe_test_util.h1
-rw-r--r--storage/src/tests/distributor/distributortestutil.cpp512
-rw-r--r--storage/src/tests/distributor/distributortestutil.h248
-rw-r--r--storage/src/tests/distributor/legacy_bucket_db_updater_test.cpp2893
-rw-r--r--storage/src/tests/distributor/legacy_distributor_test.cpp1326
-rw-r--r--storage/src/tests/distributor/statusreporterdelegatetest.cpp2
-rw-r--r--storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp24
-rw-r--r--storage/src/tests/distributor/top_level_distributor_test_util.cpp5
-rw-r--r--storage/src/tests/distributor/top_level_distributor_test_util.h1
11 files changed, 13 insertions, 5007 deletions
diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt
index 67a7fed8d0b..5a5422156d1 100644
--- a/storage/src/tests/distributor/CMakeLists.txt
+++ b/storage/src/tests/distributor/CMakeLists.txt
@@ -15,15 +15,12 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST
distributor_stripe_pool_test.cpp
distributor_stripe_test.cpp
distributor_stripe_test_util.cpp
- distributortestutil.cpp
externaloperationhandlertest.cpp
garbagecollectiontest.cpp
getoperationtest.cpp
gtest_runner.cpp
idealstatemanagertest.cpp
joinbuckettest.cpp
- legacy_bucket_db_updater_test.cpp
- legacy_distributor_test.cpp
maintenanceschedulertest.cpp
mergelimitertest.cpp
mergeoperationtest.cpp
diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.cpp b/storage/src/tests/distributor/distributor_stripe_test_util.cpp
index 043f996a4a1..4b7e73d3e43 100644
--- a/storage/src/tests/distributor/distributor_stripe_test_util.cpp
+++ b/storage/src/tests/distributor/distributor_stripe_test_util.cpp
@@ -20,7 +20,6 @@ namespace storage::distributor {
DistributorStripeTestUtil::DistributorStripeTestUtil()
: _config(),
_node(),
- _threadPool(),
_stripe(),
_sender(),
_senderDown(),
@@ -37,18 +36,14 @@ void
DistributorStripeTestUtil::createLinks()
{
_node.reset(new TestDistributorApp(_config.getConfigId()));
- _threadPool = framework::TickingThreadPool::createDefault("distributor");
_metrics = std::make_shared<DistributorMetricSet>();
_ideal_state_metrics = std::make_shared<IdealStateMetricSet>();
_stripe = std::make_unique<DistributorStripe>(_node->getComponentRegister(),
*_metrics,
*_ideal_state_metrics,
_node->node_identity(),
- *_threadPool,
- *this,
_messageSender,
*this,
- false,
_done_initializing);
}
diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.h b/storage/src/tests/distributor/distributor_stripe_test_util.h
index ccade98fd01..e73d1a3baa1 100644
--- a/storage/src/tests/distributor/distributor_stripe_test_util.h
+++ b/storage/src/tests/distributor/distributor_stripe_test_util.h
@@ -212,7 +212,6 @@ public:
protected:
vdstestlib::DirConfig _config;
std::unique_ptr<TestDistributorApp> _node;
- std::unique_ptr<framework::TickingThreadPool> _threadPool;
std::shared_ptr<DistributorMetricSet> _metrics;
std::shared_ptr<IdealStateMetricSet> _ideal_state_metrics;
std::unique_ptr<DistributorStripe> _stripe;
diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp
deleted file mode 100644
index 1e24bd72c9b..00000000000
--- a/storage/src/tests/distributor/distributortestutil.cpp
+++ /dev/null
@@ -1,512 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include "distributortestutil.h"
-#include <vespa/config-stor-distribution.h>
-#include <vespa/document/test/make_bucket_space.h>
-#include <vespa/document/test/make_document_bucket.h>
-#include <vespa/storage/distributor/top_level_distributor.h>
-#include <vespa/storage/distributor/distributor_bucket_space.h>
-#include <vespa/storage/distributor/distributor_stripe.h>
-#include <vespa/storage/distributor/distributor_stripe_component.h>
-#include <vespa/storage/distributor/distributor_stripe_pool.h>
-#include <vespa/vdslib/distribution/distribution.h>
-#include <vespa/vespalib/text/stringtokenizer.h>
-
-using document::test::makeBucketSpace;
-using document::test::makeDocumentBucket;
-
-namespace storage::distributor {
-
-DistributorTestUtil::DistributorTestUtil()
- : _messageSender(_sender, _senderDown),
- _num_distributor_stripes(0) // TODO STRIPE change default
-{
- _config = getStandardConfig(false);
-}
-DistributorTestUtil::~DistributorTestUtil() { }
-
-void
-DistributorTestUtil::createLinks()
-{
- _node.reset(new TestDistributorApp(_config.getConfigId()));
- _threadPool = framework::TickingThreadPool::createDefault("distributor");
- _stripe_pool = std::make_unique<DistributorStripePool>();
- _distributor.reset(new TopLevelDistributor(
- _node->getComponentRegister(),
- _node->node_identity(),
- *_threadPool,
- *_stripe_pool,
- *this,
- _num_distributor_stripes,
- _hostInfo,
- &_messageSender));
- _component.reset(new storage::DistributorComponent(_node->getComponentRegister(), "distrtestutil"));
-};
-
-void
-DistributorTestUtil::setupDistributor(int redundancy,
- int nodeCount,
- const std::string& systemState,
- uint32_t earlyReturn,
- bool requirePrimaryToBeWritten)
-{
- setup_distributor(redundancy, nodeCount, lib::ClusterStateBundle(lib::ClusterState(systemState)), earlyReturn, requirePrimaryToBeWritten);
-}
-
-void
-DistributorTestUtil::setup_distributor(int redundancy,
- int node_count,
- const lib::ClusterStateBundle& state,
- uint32_t early_return,
- bool require_primary_to_be_written)
-{
- lib::Distribution::DistributionConfigBuilder config(
- lib::Distribution::getDefaultDistributionConfig(redundancy, node_count).get());
- config.redundancy = redundancy;
- config.initialRedundancy = early_return;
- config.ensurePrimaryPersisted = require_primary_to_be_written;
- auto distribution = std::make_shared<lib::Distribution>(config);
- _node->getComponentRegister().setDistribution(distribution);
- enable_distributor_cluster_state(state);
- // This is for all intents and purposes a hack to avoid having the
- // distributor treat setting the distribution explicitly as a signal that
- // it should send RequestBucketInfo to all configured nodes.
- // If we called storage_distribution_changed followed by enableDistribution
- // explicitly (which is what happens in "real life"), that is what would
- // take place.
- // The inverse case of this can be explicitly accomplished by calling
- // triggerDistributionChange().
- // This isn't pretty, folks, but it avoids breaking the world for now,
- // as many tests have implicit assumptions about this being the behavior.
- _distributor->propagateDefaultDistribution(distribution);
-}
-
-void
-DistributorTestUtil::setRedundancy(uint32_t redundancy)
-{
- auto distribution = std::make_shared<lib::Distribution>(
- lib::Distribution::getDefaultDistributionConfig(
- redundancy, 100));
- // Same rationale for not triggering a full distribution change as
- // in setupDistributor()
- _node->getComponentRegister().setDistribution(distribution);
- _distributor->propagateDefaultDistribution(std::move(distribution));
-}
-
-void
-DistributorTestUtil::triggerDistributionChange(lib::Distribution::SP distr)
-{
- _node->getComponentRegister().setDistribution(std::move(distr));
- _distributor->storageDistributionChanged();
- _distributor->enableNextDistribution();
-}
-
-void
-DistributorTestUtil::receive_set_system_state_command(const vespalib::string& state_str)
-{
- auto state_cmd = std::make_shared<api::SetSystemStateCommand>(lib::ClusterState(state_str));
- _distributor->handleMessage(state_cmd); // TODO move semantics
-}
-
-void
-DistributorTestUtil::handle_top_level_message(const std::shared_ptr<api::StorageMessage>& msg)
-{
- _distributor->handleMessage(msg);
-}
-
-void
-DistributorTestUtil::setTypeRepo(const std::shared_ptr<const document::DocumentTypeRepo> &repo)
-{
- _node->getComponentRegister().setDocumentTypeRepo(repo);
-}
-
-void
-DistributorTestUtil::close()
-{
- _component.reset(0);
- if (_distributor.get()) {
- _distributor->onClose();
- }
- _sender.clear();
- _node.reset(0);
- _config = getStandardConfig(false);
-}
-
-namespace {
- std::string dumpVector(const std::vector<uint16_t>& vec) {
- std::ostringstream ost;
- for (uint32_t i = 0; i < vec.size(); ++i) {
- if (i != 0) {
- ost << ",";
- }
- ost << vec[i];
- }
- return ost.str();
- }
-}
-
-std::string
-DistributorTestUtil::getNodes(document::BucketId id)
-{
- BucketDatabase::Entry entry = getBucket(id);
-
- if (!entry.valid()) {
- return id.toString();
- } else {
- std::vector<uint16_t> nodes = entry->getNodes();
- std::sort(nodes.begin(), nodes.end());
-
- std::ostringstream ost;
- ost << id << ": " << dumpVector(nodes);
- return ost.str();
- }
-}
-
-std::string
-DistributorTestUtil::getIdealStr(document::BucketId id, const lib::ClusterState& state)
-{
- if (!getDistributorBucketSpace().owns_bucket_in_state(state, id)) {
- return id.toString();
- }
-
- std::vector<uint16_t> nodes;
- getDistribution().getIdealNodes(
- lib::NodeType::STORAGE, state, id, nodes);
- std::sort(nodes.begin(), nodes.end());
- std::ostringstream ost;
- ost << id << ": " << dumpVector(nodes);
- return ost.str();
-}
-
-void
-DistributorTestUtil::addIdealNodes(const lib::ClusterState& state,
- const document::BucketId& id)
-{
- BucketDatabase::Entry entry = getBucket(id);
-
- if (!entry.valid()) {
- entry = BucketDatabase::Entry(id);
- }
-
- std::vector<uint16_t> res;
- assert(_component.get());
- getDistribution().getIdealNodes(
- lib::NodeType::STORAGE, state, id, res);
-
- for (uint32_t i = 0; i < res.size(); ++i) {
- if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() !=
- lib::State::MAINTENANCE)
- {
- entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)),
- toVector<uint16_t>(0));
- }
- }
-
- getBucketDatabase().update(entry);
-}
-
-void DistributorTestUtil::addNodesToBucketDB(const document::Bucket& bucket, const std::string& nodeStr) {
- BucketDatabase::Entry entry = getBucket(bucket);
-
- if (!entry.valid()) {
- entry = BucketDatabase::Entry(bucket.getBucketId());
- }
-
- entry->clear();
-
- vespalib::StringTokenizer tokenizer(nodeStr, ",");
- for (uint32_t i = 0; i < tokenizer.size(); ++i) {
- vespalib::StringTokenizer tok2(tokenizer[i], "=");
- vespalib::StringTokenizer tok3(tok2[1], "/");
-
- api::BucketInfo info(atoi(tok3[0].data()),
- atoi(tok3.size() > 1 ? tok3[1].data() : tok3[0].data()),
- atoi(tok3.size() > 2 ? tok3[2].data() : tok3[0].data()));
-
- size_t flagsIdx = 3;
-
- // Meta info override? For simplicity, require both meta count and size
- if (tok3.size() > 4 && (!tok3[3].empty() && isdigit(tok3[3][0]))) {
- info.setMetaCount(atoi(tok3[3].data()));
- info.setUsedFileSize(atoi(tok3[4].data()));
- flagsIdx = 5;
- }
-
- if ((tok3.size() > flagsIdx + 1) && tok3[flagsIdx + 1] == "a") {
- info.setActive();
- } else {
- info.setActive(false);
- }
- if ((tok3.size() > flagsIdx + 2) && tok3[flagsIdx + 2] == "r") {
- info.setReady();
- } else {
- info.setReady(false);
- }
-
- uint16_t idx = atoi(tok2[0].data());
- BucketCopy node(
- 0,
- idx,
- info);
-
- // Allow user to manually override trusted and active.
- if (tok3.size() > flagsIdx && tok3[flagsIdx] == "t") {
- node.setTrusted();
- }
-
- entry->addNodeManual(node);
- }
-
- getBucketDatabase(bucket.getBucketSpace()).update(entry);
-}
-
-void
-DistributorTestUtil::addNodesToBucketDB(const document::BucketId& id,
- const std::string& nodeStr)
-{
- addNodesToBucketDB(document::Bucket(makeBucketSpace(), id), nodeStr);
-}
-
-void
-DistributorTestUtil::removeFromBucketDB(const document::BucketId& id)
-{
- getBucketDatabase().remove(id);
-}
-
-void
-DistributorTestUtil::addIdealNodes(const document::BucketId& id)
-{
- // TODO STRIPE roundabout way of getting state bundle..!
- addIdealNodes(*operation_context().cluster_state_bundle().getBaselineClusterState(), id);
-}
-
-void
-DistributorTestUtil::insertBucketInfo(document::BucketId id,
- uint16_t node,
- uint32_t checksum,
- uint32_t count,
- uint32_t size,
- bool trusted,
- bool active)
-{
- api::BucketInfo info(checksum, count, size);
- insertBucketInfo(id, node, info, trusted, active);
-}
-
-void
-DistributorTestUtil::insertBucketInfo(document::BucketId id,
- uint16_t node,
- const api::BucketInfo& info,
- bool trusted,
- bool active)
-{
- BucketDatabase::Entry entry = getBucketDatabase().get(id);
- if (!entry.valid()) {
- entry = BucketDatabase::Entry(id, BucketInfo());
- }
-
- api::BucketInfo info2(info);
- if (active) {
- info2.setActive();
- }
- BucketCopy copy(operation_context().generate_unique_timestamp(), node, info2);
-
- entry->addNode(copy.setTrusted(trusted), toVector<uint16_t>(0));
-
- getBucketDatabase().update(entry);
-}
-
-std::string
-DistributorTestUtil::dumpBucket(const document::BucketId& bid)
-{
- return getBucketDatabase().get(bid).toString();
-}
-
-void
-DistributorTestUtil::sendReply(Operation& op,
- int idx,
- api::ReturnCode::Result result)
-{
- if (idx == -1) {
- idx = _sender.commands().size() - 1;
- }
- assert(idx >= 0 && idx < static_cast<int>(_sender.commands().size()));
-
- std::shared_ptr<api::StorageCommand> cmd = _sender.command(idx);
- api::StorageReply::SP reply(cmd->makeReply().release());
- reply->setResult(result);
- op.receive(_sender, reply);
-}
-
-BucketDatabase::Entry DistributorTestUtil::getBucket(const document::Bucket& bucket) const {
- return getBucketDatabase(bucket.getBucketSpace()).get(bucket.getBucketId());
-}
-
-BucketDatabase::Entry
-DistributorTestUtil::getBucket(const document::BucketId& bId) const
-{
- return getBucketDatabase().get(bId);
-}
-
-void
-DistributorTestUtil::disableBucketActivationInConfig(bool disable)
-{
- vespa::config::content::core::StorDistributormanagerConfigBuilder config;
- config.disableBucketActivation = disable;
- getConfig().configure(config);
-}
-
-StripeBucketDBUpdater&
-DistributorTestUtil::getBucketDBUpdater() {
- return _distributor->bucket_db_updater();
-}
-IdealStateManager&
-DistributorTestUtil::getIdealStateManager() {
- return _distributor->ideal_state_manager();
-}
-ExternalOperationHandler&
-DistributorTestUtil::getExternalOperationHandler() {
- return _distributor->external_operation_handler();
-}
-
-const storage::distributor::DistributorNodeContext&
-DistributorTestUtil::node_context() const {
- return _distributor->distributor_component();
-}
-
-storage::distributor::DistributorStripeOperationContext&
-DistributorTestUtil::operation_context() {
- return _distributor->distributor_component();
-}
-
-const DocumentSelectionParser&
-DistributorTestUtil::doc_selection_parser() const {
- return _distributor->distributor_component();
-}
-
-bool
-DistributorTestUtil::tick() {
- framework::ThreadWaitInfo res(
- framework::ThreadWaitInfo::NO_MORE_CRITICAL_WORK_KNOWN);
- {
- framework::TickingLockGuard lock(
- _distributor->_threadPool.freezeCriticalTicks());
- res.merge(_distributor->doCriticalTick(0));
- }
- res.merge(_distributor->doNonCriticalTick(0));
- return !res.waitWanted();
-}
-
-DistributorConfiguration&
-DistributorTestUtil::getConfig() {
- // TODO STRIPE avoid const cast
- return const_cast<DistributorConfiguration&>(_distributor->getConfig());
-}
-
-DistributorBucketSpace &
-DistributorTestUtil::getDistributorBucketSpace()
-{
- return getBucketSpaceRepo().get(makeBucketSpace());
-}
-
-BucketDatabase&
-DistributorTestUtil::getBucketDatabase() {
- return getDistributorBucketSpace().getBucketDatabase();
-}
-
-BucketDatabase& DistributorTestUtil::getBucketDatabase(document::BucketSpace space) {
- return getBucketSpaceRepo().get(space).getBucketDatabase();
-}
-
-const BucketDatabase&
-DistributorTestUtil::getBucketDatabase() const {
- return getBucketSpaceRepo().get(makeBucketSpace()).getBucketDatabase();
-}
-
-const BucketDatabase& DistributorTestUtil::getBucketDatabase(document::BucketSpace space) const {
- return getBucketSpaceRepo().get(space).getBucketDatabase();
-}
-
-DistributorBucketSpaceRepo &
-DistributorTestUtil::getBucketSpaceRepo() {
- return _distributor->getBucketSpaceRepo();
-}
-
-const DistributorBucketSpaceRepo &
-DistributorTestUtil::getBucketSpaceRepo() const {
- return _distributor->getBucketSpaceRepo();
-}
-
-DistributorBucketSpaceRepo &
-DistributorTestUtil::getReadOnlyBucketSpaceRepo() {
- return _distributor->getReadOnlyBucketSpaceRepo();
-}
-
-const DistributorBucketSpaceRepo &
-DistributorTestUtil::getReadOnlyBucketSpaceRepo() const {
- return _distributor->getReadOnlyBucketSpaceRepo();
-}
-
-bool
-DistributorTestUtil::distributor_is_in_recovery_mode() const noexcept {
- return _distributor->isInRecoveryMode();
-}
-
-const lib::ClusterStateBundle&
-DistributorTestUtil::current_distributor_cluster_state_bundle() const noexcept {
- return getDistributor().getClusterStateBundle();
-}
-
-std::string
-DistributorTestUtil::active_ideal_state_operations() const {
- return _distributor->getActiveIdealStateOperations();
-}
-
-const PendingMessageTracker&
-DistributorTestUtil::pending_message_tracker() const noexcept {
- return _distributor->getPendingMessageTracker();
-}
-
-PendingMessageTracker&
-DistributorTestUtil::pending_message_tracker() noexcept {
- return _distributor->getPendingMessageTracker();
-}
-
-std::chrono::steady_clock::duration
-DistributorTestUtil::db_memory_sample_interval() const noexcept {
- return _distributor->db_memory_sample_interval();
-}
-
-const lib::Distribution&
-DistributorTestUtil::getDistribution() const {
- return getBucketSpaceRepo().get(makeBucketSpace()).getDistribution();
-}
-
-std::vector<document::BucketSpace>
-DistributorTestUtil::getBucketSpaces() const
-{
- std::vector<document::BucketSpace> res;
- for (const auto &repo : getBucketSpaceRepo()) {
- res.push_back(repo.first);
- }
- return res;
-}
-
-void
-DistributorTestUtil::enableDistributorClusterState(vespalib::stringref state)
-{
- getBucketDBUpdater().simulate_cluster_state_bundle_activation(
- lib::ClusterStateBundle(lib::ClusterState(state)));
-}
-
-void
-DistributorTestUtil::enable_distributor_cluster_state(const lib::ClusterStateBundle& state)
-{
- getBucketDBUpdater().simulate_cluster_state_bundle_activation(state);
-}
-
-void
-DistributorTestUtil::setSystemState(const lib::ClusterState& systemState) {
- _distributor->enableClusterStateBundle(lib::ClusterStateBundle(systemState));
-}
-
-}
diff --git a/storage/src/tests/distributor/distributortestutil.h b/storage/src/tests/distributor/distributortestutil.h
deleted file mode 100644
index 9457bfeba83..00000000000
--- a/storage/src/tests/distributor/distributortestutil.h
+++ /dev/null
@@ -1,248 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include "distributor_message_sender_stub.h"
-#include <tests/common/dummystoragelink.h>
-#include <tests/common/testhelper.h>
-#include <tests/common/teststorageapp.h>
-#include <vespa/storage/common/hostreporter/hostinfo.h>
-#include <vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h>
-#include <vespa/storage/storageutil/utils.h>
-#include <vespa/storageapi/message/state.h>
-#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h>
-
-namespace storage {
-
-namespace framework { struct TickingThreadPool; }
-
-namespace distributor {
-
-class TopLevelDistributor;
-class DistributorBucketSpace;
-class DistributorBucketSpaceRepo;
-class DistributorNodeContext;
-class DistributorStripe;
-class DistributorStripeComponent;
-class DistributorStripeOperationContext;
-class DistributorStripePool;
-class DocumentSelectionParser;
-class ExternalOperationHandler;
-class IdealStateManager;
-class Operation;
-class StripeBucketDBUpdater;
-
-// TODO STRIPE rename to DistributorStripeTestUtil?
-class DistributorTestUtil : private DoneInitializeHandler
-{
-public:
- DistributorTestUtil();
- ~DistributorTestUtil();
-
- /**
- * Sets up the storage link chain.
- */
- void createLinks();
- void setTypeRepo(const std::shared_ptr<const document::DocumentTypeRepo> &repo);
-
- void close();
-
- /**
- * Returns a string with the nodes currently stored in the bucket
- * database for the given bucket.
- */
- std::string getNodes(document::BucketId id);
-
- /**
- * Returns a string with the ideal state nodes for the given bucket.
- */
- std::string getIdealStr(document::BucketId id, const lib::ClusterState& state);
-
- /**
- * Adds the ideal nodes for the given bucket and the given cluster state
- * to the bucket database.
- */
- void addIdealNodes(const lib::ClusterState& state, const document::BucketId& id);
-
- /**
- * Adds all the ideal nodes for the given bucket to the bucket database.
- */
- void addIdealNodes(const document::BucketId& id);
-
- /**
- * Parses the given string to a set of node => bucket info data,
- * and inserts them as nodes in the given bucket.
- * Format:
- * "node1=checksum/docs/size,node2=checksum/docs/size"
- */
- void addNodesToBucketDB(const document::Bucket& bucket, const std::string& nodeStr);
- // As the above, but always inserts into default bucket space
- void addNodesToBucketDB(const document::BucketId& id, const std::string& nodeStr);
-
- /**
- * Removes the given bucket from the bucket database.
- */
- void removeFromBucketDB(const document::BucketId& id);
-
- /**
- * Inserts the given bucket information for the given bucket and node in
- * the bucket database.
- */
- void insertBucketInfo(document::BucketId id,
- uint16_t node,
- uint32_t checksum,
- uint32_t count,
- uint32_t size,
- bool trusted = false,
- bool active = false);
-
- /**
- * Inserts the given bucket information for the given bucket and node in
- * the bucket database.
- */
- void insertBucketInfo(document::BucketId id,
- uint16_t node,
- const api::BucketInfo& info,
- bool trusted = false,
- bool active = false);
-
- std::string dumpBucket(const document::BucketId& bucket);
-
- /**
- * Replies to message idx sent upwards with the given result code.
- * If idx = -1, replies to the last command received upwards.
- */
- void sendReply(Operation& op,
- int idx = -1,
- api::ReturnCode::Result result = api::ReturnCode::OK);
-
- StripeBucketDBUpdater& getBucketDBUpdater();
- IdealStateManager& getIdealStateManager();
- ExternalOperationHandler& getExternalOperationHandler();
- const storage::distributor::DistributorNodeContext& node_context() const;
- storage::distributor::DistributorStripeOperationContext& operation_context();
- const DocumentSelectionParser& doc_selection_parser() const;
-
- TopLevelDistributor& getDistributor() noexcept { return *_distributor; }
- const TopLevelDistributor& getDistributor() const noexcept { return *_distributor; }
-
- bool tick();
-
- DistributorConfiguration& getConfig();
-
- vdstestlib::DirConfig& getDirConfig() {
- return _config;
- }
-
- // TODO explicit notion of bucket spaces for tests
- DistributorBucketSpace &getDistributorBucketSpace();
- BucketDatabase& getBucketDatabase(); // Implicit default space only
- BucketDatabase& getBucketDatabase(document::BucketSpace space);
- const BucketDatabase& getBucketDatabase() const; // Implicit default space only
- const BucketDatabase& getBucketDatabase(document::BucketSpace space) const;
- DistributorBucketSpaceRepo &getBucketSpaceRepo();
- const DistributorBucketSpaceRepo &getBucketSpaceRepo() const;
- DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo();
- const DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo() const;
- [[nodiscard]] bool distributor_is_in_recovery_mode() const noexcept;
- [[nodiscard]] const lib::ClusterStateBundle& current_distributor_cluster_state_bundle() const noexcept;
- [[nodiscard]] std::string active_ideal_state_operations() const;
- [[nodiscard]] const PendingMessageTracker& pending_message_tracker() const noexcept;
- [[nodiscard]] PendingMessageTracker& pending_message_tracker() noexcept;
- [[nodiscard]] std::chrono::steady_clock::duration db_memory_sample_interval() const noexcept;
-
- const lib::Distribution& getDistribution() const;
- // "End to end" distribution change trigger, which will invoke the bucket
- // DB updater as expected based on the previous and new cluster state
- // and config.
- void triggerDistributionChange(std::shared_ptr<lib::Distribution> distr);
-
- framework::defaultimplementation::FakeClock& getClock() { return _node->getClock(); }
- DistributorComponentRegister& getComponentRegister() { return _node->getComponentRegister(); }
- DistributorComponentRegisterImpl& getComponentRegisterImpl() { return _node->getComponentRegister(); }
-
- StorageComponent& getComponent() {
- if (_component.get() == 0) {
- _component.reset(new storage::DistributorComponent(
- _node->getComponentRegister(), "distributor_test_utils"));
- }
- return *_component;
- }
-
- void setupDistributor(int redundancy,
- int nodeCount,
- const std::string& systemState,
- uint32_t earlyReturn = false,
- bool requirePrimaryToBeWritten = true);
-
- void setup_distributor(int redundancy,
- int node_count,
- const lib::ClusterStateBundle& state,
- uint32_t early_return = false,
- bool require_primary_to_be_written = true);
-
- void setRedundancy(uint32_t redundancy);
-
- void notifyDoneInitializing() override {}
-
- // Must implement this for storage server interface for now
- virtual api::Timestamp getUniqueTimestamp() {
- return _component->getUniqueTimestamp();
- }
-
- void disableBucketActivationInConfig(bool disable);
-
- BucketDatabase::Entry getBucket(const document::Bucket& bucket) const;
- // Gets bucket entry from default space only
- BucketDatabase::Entry getBucket(const document::BucketId& bId) const;
-
- std::vector<document::BucketSpace> getBucketSpaces() const;
-
- DistributorMessageSenderStub& sender() noexcept { return _sender; }
- const DistributorMessageSenderStub& sender() const noexcept { return _sender; }
-
- void setSystemState(const lib::ClusterState& systemState);
-
- // Invokes full cluster state transition pipeline rather than directly applying
- // the state and just pretending everything has been completed.
- void receive_set_system_state_command(const vespalib::string& state_str);
-
- void handle_top_level_message(const std::shared_ptr<api::StorageMessage>& msg);
-
- // Must be called prior to createLinks() to have any effect
- void set_num_distributor_stripes(uint32_t n_stripes) noexcept {
- _num_distributor_stripes = n_stripes;
- }
-protected:
- vdstestlib::DirConfig _config;
- std::unique_ptr<TestDistributorApp> _node;
- std::unique_ptr<framework::TickingThreadPool> _threadPool;
- std::unique_ptr<DistributorStripePool> _stripe_pool;
- std::unique_ptr<TopLevelDistributor> _distributor;
- std::unique_ptr<storage::DistributorComponent> _component;
- DistributorMessageSenderStub _sender;
- DistributorMessageSenderStub _senderDown;
- HostInfo _hostInfo;
-
- struct MessageSenderImpl : public ChainedMessageSender {
- DistributorMessageSenderStub& _sender;
- DistributorMessageSenderStub& _senderDown;
- MessageSenderImpl(DistributorMessageSenderStub& up, DistributorMessageSenderStub& down)
- : _sender(up), _senderDown(down) {}
-
- void sendUp(const std::shared_ptr<api::StorageMessage>& msg) override {
- _sender.send(msg);
- }
- void sendDown(const std::shared_ptr<api::StorageMessage>& msg) override {
- _senderDown.send(msg);
- }
- };
- MessageSenderImpl _messageSender;
- uint32_t _num_distributor_stripes;
-
- void enableDistributorClusterState(vespalib::stringref state);
- void enable_distributor_cluster_state(const lib::ClusterStateBundle& state);
-};
-
-}
-
-}
diff --git a/storage/src/tests/distributor/legacy_bucket_db_updater_test.cpp b/storage/src/tests/distributor/legacy_bucket_db_updater_test.cpp
deleted file mode 100644
index 5fa3ae5840b..00000000000
--- a/storage/src/tests/distributor/legacy_bucket_db_updater_test.cpp
+++ /dev/null
@@ -1,2893 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/storageapi/message/persistence.h>
-#include <vespa/storage/distributor/top_level_bucket_db_updater.h>
-#include <vespa/storage/distributor/bucket_space_distribution_context.h>
-#include <vespa/storage/distributor/distributormetricsset.h>
-#include <vespa/storage/distributor/pending_bucket_space_db_transition.h>
-#include <vespa/storage/distributor/outdated_nodes_map.h>
-#include <vespa/storage/storageutil/distributorstatecache.h>
-#include <tests/distributor/distributortestutil.h>
-#include <vespa/document/test/make_document_bucket.h>
-#include <vespa/document/test/make_bucket_space.h>
-#include <vespa/document/bucket/fixed_bucket_spaces.h>
-#include <vespa/storage/distributor/simpleclusterinformation.h>
-#include <vespa/storage/distributor/top_level_distributor.h>
-#include <vespa/storage/distributor/distributor_stripe.h>
-#include <vespa/storage/distributor/distributor_bucket_space.h>
-#include <vespa/vespalib/gtest/gtest.h>
-#include <vespa/vespalib/text/stringtokenizer.h>
-#include <vespa/vespalib/util/benchmark_timer.h>
-#include <sstream>
-#include <iomanip>
-
-using namespace storage::api;
-using namespace storage::lib;
-using document::test::makeDocumentBucket;
-using document::test::makeBucketSpace;
-using document::BucketSpace;
-using document::FixedBucketSpaces;
-using document::BucketId;
-using document::Bucket;
-
-using namespace ::testing;
-
-namespace storage::distributor {
-
-namespace {
-
-std::string
-getStringList(std::string s, uint32_t count)
-{
- std::ostringstream ost;
- for (uint32_t i = 0; i < count; ++i) {
- if (i > 0) {
- ost << ",";
- }
- ost << s;
- }
- return ost.str();
-}
-
-std::string
-getRequestBucketInfoStrings(uint32_t count)
-{
- return getStringList("Request bucket info", count);
-}
-
-}
-
-// TODO STRIPE: Remove this test when legacy mode is gone.
-class LegacyBucketDBUpdaterTest : public Test,
- public DistributorTestUtil
-{
-public:
- LegacyBucketDBUpdaterTest();
- ~LegacyBucketDBUpdaterTest() override;
-
- auto &defaultDistributorBucketSpace() { return getBucketSpaceRepo().get(makeBucketSpace()); }
-
- bool bucketExistsThatHasNode(int bucketCount, uint16_t node) const;
-
- ClusterInformation::CSP createClusterInfo(const std::string& clusterStateString) {
- lib::ClusterState baselineClusterState(clusterStateString);
- lib::ClusterStateBundle clusterStateBundle(baselineClusterState);
- ClusterInformation::CSP clusterInfo(
- new SimpleClusterInformation(
- getBucketDBUpdater().node_context().node_index(),
- clusterStateBundle,
- "ui"));
- for (auto* repo : {&mutable_repo(), &read_only_repo()}) {
- for (auto& space : *repo) {
- space.second->setClusterState(clusterStateBundle.getDerivedClusterState(space.first));
- }
- }
- return clusterInfo;
- }
-
- DistributorBucketSpaceRepo& mutable_repo() noexcept { return getBucketSpaceRepo(); }
- // Note: not calling this "immutable_repo" since it may actually be modified by the pending
- // cluster state component (just not by operations), so it would not have the expected semantics.
- DistributorBucketSpaceRepo& read_only_repo() noexcept { return getReadOnlyBucketSpaceRepo(); }
-
- BucketDatabase& mutable_default_db() noexcept {
- return mutable_repo().get(FixedBucketSpaces::default_space()).getBucketDatabase();
- }
- BucketDatabase& mutable_global_db() noexcept {
- return mutable_repo().get(FixedBucketSpaces::global_space()).getBucketDatabase();
- }
- BucketDatabase& read_only_default_db() noexcept {
- return read_only_repo().get(FixedBucketSpaces::default_space()).getBucketDatabase();
- }
- BucketDatabase& read_only_global_db() noexcept {
- return read_only_repo().get(FixedBucketSpaces::global_space()).getBucketDatabase();
- }
-
- static std::string getNodeList(std::vector<uint16_t> nodes, size_t count);
-
- std::string getNodeList(std::vector<uint16_t> nodes);
-
- std::vector<uint16_t>
- expandNodeVec(const std::vector<uint16_t> &nodes);
-
- std::vector<document::BucketSpace> _bucketSpaces;
-
- size_t messageCount(size_t messagesPerBucketSpace) const {
- return messagesPerBucketSpace * _bucketSpaces.size();
- }
-
- void trigger_completed_but_not_yet_activated_transition(
- vespalib::stringref initial_state, uint32_t initial_buckets, uint32_t initial_expected_msgs,
- vespalib::stringref pending_state, uint32_t pending_buckets, uint32_t pending_expected_msgs);
-
-public:
- using OutdatedNodesMap = dbtransition::OutdatedNodesMap;
- void SetUp() override {
- createLinks();
- _bucketSpaces = getBucketSpaces();
- // Disable deferred activation by default (at least for now) to avoid breaking the entire world.
- getBucketDBUpdater().set_stale_reads_enabled(false);
- };
-
- void TearDown() override {
- close();
- }
-
- std::shared_ptr<RequestBucketInfoReply> getFakeBucketReply(
- const lib::ClusterState& state,
- const RequestBucketInfoCommand& cmd,
- int storageIndex,
- uint32_t bucketCount,
- uint32_t invalidBucketCount = 0)
- {
- auto sreply = std::make_shared<RequestBucketInfoReply>(cmd);
- sreply->setAddress(storageAddress(storageIndex));
-
- api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo();
-
- for (uint32_t i=0; i<bucketCount + invalidBucketCount; i++) {
- if (!getDistributorBucketSpace().owns_bucket_in_state(state, document::BucketId(16, i))) {
- continue;
- }
-
- std::vector<uint16_t> nodes;
- defaultDistributorBucketSpace().getDistribution().getIdealNodes(
- lib::NodeType::STORAGE,
- state,
- document::BucketId(16, i),
- nodes);
-
- for (uint32_t j=0; j<nodes.size(); j++) {
- if (nodes[j] == storageIndex) {
- if (i >= bucketCount) {
- vec.push_back(api::RequestBucketInfoReply::Entry(
- document::BucketId(16, i),
- api::BucketInfo()));
- } else {
- vec.push_back(api::RequestBucketInfoReply::Entry(
- document::BucketId(16, i),
- api::BucketInfo(10,1,1)));
- }
- }
- }
- }
-
- return sreply;
- }
-
- void fakeBucketReply(const lib::ClusterState &state,
- const api::StorageCommand &cmd,
- uint32_t bucketCount,
- uint32_t invalidBucketCount = 0)
- {
- ASSERT_EQ(cmd.getType(), MessageType::REQUESTBUCKETINFO);
- const api::StorageMessageAddress &address(*cmd.getAddress());
- getBucketDBUpdater().onRequestBucketInfoReply(
- getFakeBucketReply(state,
- dynamic_cast<const RequestBucketInfoCommand &>(cmd),
- address.getIndex(),
- bucketCount,
- invalidBucketCount));
- }
-
- void sendFakeReplyForSingleBucketRequest(
- const api::RequestBucketInfoCommand& rbi)
- {
- ASSERT_EQ(size_t(1), rbi.getBuckets().size());
- const document::BucketId& bucket(rbi.getBuckets()[0]);
-
- std::shared_ptr<api::RequestBucketInfoReply> reply(
- new api::RequestBucketInfoReply(rbi));
- reply->getBucketInfo().push_back(
- api::RequestBucketInfoReply::Entry(bucket,
- api::BucketInfo(20, 10, 12, 50, 60, true, true)));
- getBucketDBUpdater().onRequestBucketInfoReply(reply);
- }
-
- std::string verifyBucket(document::BucketId id, const lib::ClusterState& state) {
- BucketDatabase::Entry entry = getBucketDatabase().get(id);
- if (!entry.valid()) {
- return vespalib::make_string("%s doesn't exist in DB",
- id.toString().c_str());
- }
-
- std::vector<uint16_t> nodes;
- defaultDistributorBucketSpace().getDistribution().getIdealNodes(
- lib::NodeType::STORAGE,
- state,
- document::BucketId(id),
- nodes);
-
- if (nodes.size() != entry->getNodeCount()) {
- return vespalib::make_string("Bucket Id %s has %d nodes in "
- "ideal state, but has only %d in DB",
- id.toString().c_str(),
- (int)nodes.size(),
- (int)entry->getNodeCount());
- }
-
- for (uint32_t i = 0; i<nodes.size(); i++) {
- bool found = false;
-
- for (uint32_t j = 0; j<entry->getNodeCount(); j++) {
- if (nodes[i] == entry->getNodeRef(j).getNode()) {
- found = true;
- }
- }
-
- if (!found) {
- return vespalib::make_string(
- "Bucket Id %s has no copy from node %d",
- id.toString().c_str(),
- nodes[i]);
- }
- }
-
- return "";
- }
-
-
- void verifyInvalid(document::BucketId id, int storageNode) {
- BucketDatabase::Entry entry = getBucketDatabase().get(id);
-
- ASSERT_TRUE(entry.valid());
-
- bool found = false;
- for (uint32_t j = 0; j<entry->getNodeCount(); j++) {
- if (entry->getNodeRef(j).getNode() == storageNode) {
- ASSERT_FALSE(entry->getNodeRef(j).valid());
- found = true;
- }
- }
-
- ASSERT_TRUE(found);
- }
-
- struct OrderByIncreasingNodeIndex {
- template <typename T>
- bool operator()(const T& lhs, const T& rhs) {
- return (lhs->getAddress()->getIndex()
- < rhs->getAddress()->getIndex());
- }
- };
-
- void sortSentMessagesByIndex(DistributorMessageSenderStub& sender,
- size_t sortFromOffset = 0)
- {
- std::sort(sender.commands().begin() + sortFromOffset,
- sender.commands().end(),
- OrderByIncreasingNodeIndex());
- }
-
- void setSystemState(const lib::ClusterState& state) {
- const size_t sizeBeforeState = _sender.commands().size();
- getBucketDBUpdater().onSetSystemState(
- std::make_shared<api::SetSystemStateCommand>(state));
- // A lot of test logic has the assumption that all messages sent as a
- // result of cluster state changes will be in increasing index order
- // (for simplicity, not because this is required for correctness).
- // Only sort the messages that arrived as a result of the state, don't
- // jumble the sorting with any existing messages.
- sortSentMessagesByIndex(_sender, sizeBeforeState);
- }
-
- void set_cluster_state_bundle(const lib::ClusterStateBundle& state) {
- const size_t sizeBeforeState = _sender.commands().size();
- getBucketDBUpdater().onSetSystemState(
- std::make_shared<api::SetSystemStateCommand>(state));
- sortSentMessagesByIndex(_sender, sizeBeforeState);
- }
-
- bool activate_cluster_state_version(uint32_t version) {
- return getBucketDBUpdater().onActivateClusterStateVersion(
- std::make_shared<api::ActivateClusterStateVersionCommand>(version));
- }
-
- void assert_has_activate_cluster_state_reply_with_actual_version(uint32_t version) {
- ASSERT_EQ(size_t(1), _sender.replies().size());
- auto* response = dynamic_cast<api::ActivateClusterStateVersionReply*>(_sender.replies().back().get());
- ASSERT_TRUE(response != nullptr);
- ASSERT_EQ(version, response->actualVersion());
- _sender.clear();
- }
-
- void completeBucketInfoGathering(const lib::ClusterState& state,
- size_t expectedMsgs,
- uint32_t bucketCount = 1,
- uint32_t invalidBucketCount = 0)
- {
- ASSERT_EQ(expectedMsgs, _sender.commands().size());
-
- for (uint32_t i = 0; i < _sender.commands().size(); i++) {
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(state, *_sender.command(i),
- bucketCount, invalidBucketCount));
- }
- }
-
- void setAndEnableClusterState(const lib::ClusterState& state,
- uint32_t expectedMsgs,
- uint32_t nBuckets)
- {
- _sender.clear();
- setSystemState(state);
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(state, expectedMsgs, nBuckets));
- }
-
- void completeStateTransitionInSeconds(const std::string& stateStr,
- uint32_t seconds,
- uint32_t expectedMsgs)
- {
- _sender.clear();
- lib::ClusterState state(stateStr);
- setSystemState(state);
- getClock().addSecondsToTime(seconds);
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(state, expectedMsgs));
- }
-
- uint64_t lastTransitionTimeInMillis() {
- return uint64_t(getDistributor().getMetrics().stateTransitionTime.getLast());
- }
-
- void setStorageNodes(uint32_t numStorageNodes) {
- _sender.clear();
-
- lib::ClusterState newState(
- vespalib::make_string("distributor:1 storage:%d", numStorageNodes));
-
- setSystemState(newState);
-
- for (uint32_t i=0; i< messageCount(numStorageNodes); i++) {
- ASSERT_EQ(_sender.command(i)->getType(), MessageType::REQUESTBUCKETINFO);
-
- const api::StorageMessageAddress *address = _sender.command(i)->getAddress();
- ASSERT_EQ((uint32_t)(i / _bucketSpaces.size()), (uint32_t)address->getIndex());
- }
- }
-
- void initializeNodesAndBuckets(uint32_t numStorageNodes,
- uint32_t numBuckets)
- {
- ASSERT_NO_FATAL_FAILURE(setStorageNodes(numStorageNodes));
-
- vespalib::string state(vespalib::make_string(
- "distributor:1 storage:%d", numStorageNodes));
- lib::ClusterState newState(state);
-
- for (uint32_t i=0; i< messageCount(numStorageNodes); i++) {
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(newState, *_sender.command(i), numBuckets));
- }
- ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(numBuckets, state));
- }
-
- bool bucketHasNode(document::BucketId id, uint16_t node) const {
- BucketDatabase::Entry entry = getBucket(id);
- assert(entry.valid());
-
- for (uint32_t j=0; j<entry->getNodeCount(); j++) {
- if (entry->getNodeRef(j).getNode() == node) {
- return true;
- }
- }
-
- return false;
- }
-
- api::StorageMessageAddress storageAddress(uint16_t node) {
- static vespalib::string _storage("storage");
- return api::StorageMessageAddress(&_storage, lib::NodeType::STORAGE, node);
- }
-
- std::string getSentNodes(const std::string& oldClusterState,
- const std::string& newClusterState);
-
- std::string getSentNodesDistributionChanged(
- const std::string& oldClusterState);
-
- std::vector<uint16_t> getSentNodesWithPreemption(
- const std::string& oldClusterState,
- uint32_t expectedOldStateMessages,
- const std::string& preemptedClusterState,
- const std::string& newClusterState);
-
- std::vector<uint16_t> getSendSet() const;
-
- std::string mergeBucketLists(
- const lib::ClusterState& oldState,
- const std::string& existingData,
- const lib::ClusterState& newState,
- const std::string& newData,
- bool includeBucketInfo = false);
-
- std::string mergeBucketLists(
- const std::string& existingData,
- const std::string& newData,
- bool includeBucketInfo = false);
-
- void assertCorrectBuckets(int numBuckets, const std::string& stateStr) {
- lib::ClusterState state(stateStr);
- for (int i=0; i<numBuckets; i++) {
- ASSERT_EQ(getIdealStr(document::BucketId(16, i), state),
- getNodes(document::BucketId(16, i)));
- }
- }
-
- void setDistribution(const std::string& distConfig) {
- triggerDistributionChange(
- std::make_shared<lib::Distribution>(distConfig));
- }
-
- std::string getDistConfig6Nodes2Groups() const {
- return ("redundancy 2\n"
- "group[3]\n"
- "group[0].name \"invalid\"\n"
- "group[0].index \"invalid\"\n"
- "group[0].partitions 1|*\n"
- "group[0].nodes[0]\n"
- "group[1].name rack0\n"
- "group[1].index 0\n"
- "group[1].nodes[3]\n"
- "group[1].nodes[0].index 0\n"
- "group[1].nodes[1].index 1\n"
- "group[1].nodes[2].index 2\n"
- "group[2].name rack1\n"
- "group[2].index 1\n"
- "group[2].nodes[3]\n"
- "group[2].nodes[0].index 3\n"
- "group[2].nodes[1].index 4\n"
- "group[2].nodes[2].index 5\n");
- }
-
- std::string getDistConfig6Nodes4Groups() const {
- return ("redundancy 2\n"
- "group[4]\n"
- "group[0].name \"invalid\"\n"
- "group[0].index \"invalid\"\n"
- "group[0].partitions 1|*\n"
- "group[0].nodes[0]\n"
- "group[1].name rack0\n"
- "group[1].index 0\n"
- "group[1].nodes[2]\n"
- "group[1].nodes[0].index 0\n"
- "group[1].nodes[1].index 1\n"
- "group[2].name rack1\n"
- "group[2].index 1\n"
- "group[2].nodes[2]\n"
- "group[2].nodes[0].index 2\n"
- "group[2].nodes[1].index 3\n"
- "group[3].name rack2\n"
- "group[3].index 2\n"
- "group[3].nodes[2]\n"
- "group[3].nodes[0].index 4\n"
- "group[3].nodes[1].index 5\n");
- }
-
- std::string getDistConfig3Nodes1Group() const {
- return ("redundancy 2\n"
- "group[2]\n"
- "group[0].name \"invalid\"\n"
- "group[0].index \"invalid\"\n"
- "group[0].partitions 1|*\n"
- "group[0].nodes[0]\n"
- "group[1].name rack0\n"
- "group[1].index 0\n"
- "group[1].nodes[3]\n"
- "group[1].nodes[0].index 0\n"
- "group[1].nodes[1].index 1\n"
- "group[1].nodes[2].index 2\n");
- }
-
- struct PendingClusterStateFixture {
- DistributorMessageSenderStub sender;
- framework::defaultimplementation::FakeClock clock;
- std::unique_ptr<PendingClusterState> state;
-
- PendingClusterStateFixture(
- LegacyBucketDBUpdaterTest& owner,
- const std::string& oldClusterState,
- const std::string& newClusterState)
- {
- std::shared_ptr<api::SetSystemStateCommand> cmd(
- new api::SetSystemStateCommand(
- lib::ClusterState(newClusterState)));
-
- ClusterInformation::CSP clusterInfo(
- owner.createClusterInfo(oldClusterState));
-
- OutdatedNodesMap outdatedNodesMap;
- state = PendingClusterState::createForClusterStateChange(
- clock, clusterInfo, sender,
- owner.getBucketSpaceRepo(),
- cmd, outdatedNodesMap, api::Timestamp(1));
- }
-
- PendingClusterStateFixture(
- LegacyBucketDBUpdaterTest& owner,
- const std::string& oldClusterState)
- {
- ClusterInformation::CSP clusterInfo(
- owner.createClusterInfo(oldClusterState));
-
- state = PendingClusterState::createForDistributionChange(
- clock, clusterInfo, sender, owner.getBucketSpaceRepo(), api::Timestamp(1));
- }
- };
-
- std::unique_ptr<PendingClusterStateFixture> createPendingStateFixtureForStateChange(
- const std::string& oldClusterState,
- const std::string& newClusterState)
- {
- return std::make_unique<PendingClusterStateFixture>(*this, oldClusterState, newClusterState);
- }
-
- std::unique_ptr<PendingClusterStateFixture> createPendingStateFixtureForDistributionChange(
- const std::string& oldClusterState)
- {
- return std::make_unique<PendingClusterStateFixture>(*this, oldClusterState);
- }
-
- uint32_t populate_bucket_db_via_request_bucket_info_for_benchmarking();
-
- void complete_recovery_mode() {
- _distributor->scanAllBuckets();
- }
-};
-
-LegacyBucketDBUpdaterTest::LegacyBucketDBUpdaterTest()
- : DistributorTestUtil(),
- _bucketSpaces()
-{
-}
-
-LegacyBucketDBUpdaterTest::~LegacyBucketDBUpdaterTest() = default;
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, normal_usage) {
- setSystemState(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"));
-
- ASSERT_EQ(messageCount(3), _sender.commands().size());
-
- // Ensure distribution hash is set correctly
- ASSERT_EQ(
- defaultDistributorBucketSpace().getDistribution()
- .getNodeGraph().getDistributionConfigHash(),
- dynamic_cast<const RequestBucketInfoCommand&>(
- *_sender.command(0)).getDistributionHash());
-
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"),
- *_sender.command(0), 10));
-
- _sender.clear();
-
- // Optimization for not refetching unneeded data after cluster state
- // change is only implemented after completion of previous cluster state
- setSystemState(lib::ClusterState("distributor:2 .0.s:i storage:3"));
-
- ASSERT_EQ(messageCount(3), _sender.commands().size());
- // Expect reply of first set SystemState request.
- ASSERT_EQ(size_t(1), _sender.replies().size());
-
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(
- lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"),
- messageCount(3), 10));
- ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(10, "distributor:2 storage:3"));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, distributor_change) {
- int numBuckets = 100;
-
- // First sends request
- setSystemState(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"));
- ASSERT_EQ(messageCount(3), _sender.commands().size());
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"),
- messageCount(3), numBuckets));
- _sender.clear();
-
- // No change from initializing to up (when done with last job)
- setSystemState(lib::ClusterState("distributor:2 storage:3"));
- ASSERT_EQ(size_t(0), _sender.commands().size());
- _sender.clear();
-
- // Adding node. No new read requests, but buckets thrown
- setSystemState(lib::ClusterState("distributor:3 storage:3"));
- ASSERT_EQ(size_t(0), _sender.commands().size());
- ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(numBuckets, "distributor:3 storage:3"));
- _sender.clear();
-
- // Removing distributor. Need to refetch new data from all nodes.
- setSystemState(lib::ClusterState("distributor:2 storage:3"));
- ASSERT_EQ(messageCount(3), _sender.commands().size());
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:2 storage:3"),
- messageCount(3), numBuckets));
- _sender.clear();
- ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(numBuckets, "distributor:2 storage:3"));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, distributor_change_with_grouping) {
- std::string distConfig(getDistConfig6Nodes2Groups());
- setDistribution(distConfig);
- int numBuckets = 100;
-
- setSystemState(lib::ClusterState("distributor:6 storage:6"));
- ASSERT_EQ(messageCount(6), _sender.commands().size());
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:6 storage:6"),
- messageCount(6), numBuckets));
- _sender.clear();
-
- // Distributor going down in other group, no change
- setSystemState(lib::ClusterState("distributor:6 .5.s:d storage:6"));
- ASSERT_EQ(size_t(0), _sender.commands().size());
- _sender.clear();
-
- setSystemState(lib::ClusterState("distributor:6 storage:6"));
- ASSERT_EQ(size_t(0), _sender.commands().size());
- ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(numBuckets, "distributor:6 storage:6"));
- _sender.clear();
-
- // Unchanged grouping cause no change.
- setDistribution(distConfig);
- ASSERT_EQ(size_t(0), _sender.commands().size());
-
- // Changed grouping cause change
- setDistribution(getDistConfig6Nodes4Groups());
-
- ASSERT_EQ(messageCount(6), _sender.commands().size());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, normal_usage_initializing) {
- setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1 .0.s:i"));
-
- ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size());
-
- // Not yet passing on system state.
- ASSERT_EQ(size_t(0), _senderDown.commands().size());
-
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:1 .0.s:i storage:1"),
- _bucketSpaces.size(), 10, 10));
-
- ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(10, "distributor:1 storage:1"));
-
- for (int i=10; i<20; i++) {
- ASSERT_NO_FATAL_FAILURE(verifyInvalid(document::BucketId(16, i), 0));
- }
-
- // Pass on cluster state and recheck buckets now.
- ASSERT_EQ(size_t(1), _senderDown.commands().size());
-
- _sender.clear();
- _senderDown.clear();
-
- setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1"));
-
- // Send a new request bucket info up.
- ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size());
-
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:1 .0.s:i storage:1"),
- _bucketSpaces.size(), 20));
-
- // Pass on cluster state and recheck buckets now.
- ASSERT_EQ(size_t(1), _senderDown.commands().size());
-
- ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(20, "distributor:1 storage:1"));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, failed_request_bucket_info) {
- setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1"));
-
- // 2 messages sent up: 1 to the nodes, and one reply to the setsystemstate.
- ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size());
-
- {
- for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) {
- std::shared_ptr<api::RequestBucketInfoReply> reply =
- getFakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"),
- *((RequestBucketInfoCommand*)_sender.command(i).get()),
- 0,
- 10);
- reply->setResult(api::ReturnCode::NOT_CONNECTED);
- getBucketDBUpdater().onRequestBucketInfoReply(reply);
- }
-
- // Trigger that delayed message is sent
- getClock().addSecondsToTime(10);
- getBucketDBUpdater().resendDelayedMessages();
- }
-
- // Should be resent.
- ASSERT_EQ(getRequestBucketInfoStrings(messageCount(2)), _sender.getCommands());
-
- ASSERT_EQ(size_t(0), _senderDown.commands().size());
-
- for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) {
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"),
- *_sender.command(_bucketSpaces.size() + i), 10));
- }
-
- for (int i=0; i<10; i++) {
- EXPECT_EQ(std::string(""),
- verifyBucket(document::BucketId(16, i),
- lib::ClusterState("distributor:1 storage:1")));
- }
-
- // Set system state should now be passed on
- EXPECT_EQ(std::string("Set system state"), _senderDown.getCommands());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, down_while_init) {
- ASSERT_NO_FATAL_FAILURE(setStorageNodes(3));
-
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:1 storage:3"),
- *_sender.command(0), 5));
-
- setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:d"));
-
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:1 storage:3"),
- *_sender.command(2), 5));
-
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:1 storage:3"),
- *_sender.command(1), 5));
-}
-
-bool
-LegacyBucketDBUpdaterTest::bucketExistsThatHasNode(int bucketCount, uint16_t node) const
-{
- for (int i=1; i<bucketCount; i++) {
- if (bucketHasNode(document::BucketId(16, i), node)) {
- return true;
- }
- }
-
- return false;
-}
-
-std::string
-LegacyBucketDBUpdaterTest::getNodeList(std::vector<uint16_t> nodes, size_t count)
-{
- std::ostringstream ost;
- bool first = true;
- for (const auto &node : nodes) {
- for (uint32_t i = 0; i < count; ++i) {
- if (!first) {
- ost << ",";
- }
- ost << node;
- first = false;
- }
- }
- return ost.str();
-}
-
-std::string
-LegacyBucketDBUpdaterTest::getNodeList(std::vector<uint16_t> nodes)
-{
- return getNodeList(std::move(nodes), _bucketSpaces.size());
-}
-
-std::vector<uint16_t>
-LegacyBucketDBUpdaterTest::expandNodeVec(const std::vector<uint16_t> &nodes)
-{
- std::vector<uint16_t> res;
- size_t count = _bucketSpaces.size();
- for (const auto &node : nodes) {
- for (uint32_t i = 0; i < count; ++i) {
- res.push_back(node);
- }
- }
- return res;
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, node_down) {
- ASSERT_NO_FATAL_FAILURE(setStorageNodes(3));
- enableDistributorClusterState("distributor:1 storage:3");
-
- for (int i=1; i<100; i++) {
- addIdealNodes(document::BucketId(16, i));
- }
-
- EXPECT_TRUE(bucketExistsThatHasNode(100, 1));
-
- setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:d"));
-
- EXPECT_FALSE(bucketExistsThatHasNode(100, 1));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, storage_node_in_maintenance_clears_buckets_for_node) {
- ASSERT_NO_FATAL_FAILURE(setStorageNodes(3));
- enableDistributorClusterState("distributor:1 storage:3");
-
- for (int i=1; i<100; i++) {
- addIdealNodes(document::BucketId(16, i));
- }
-
- EXPECT_TRUE(bucketExistsThatHasNode(100, 1));
-
- setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:m"));
-
- EXPECT_FALSE(bucketExistsThatHasNode(100, 1));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, node_down_copies_get_in_sync) {
- ASSERT_NO_FATAL_FAILURE(setStorageNodes(3));
-
- lib::ClusterState systemState("distributor:1 storage:3");
- document::BucketId bid(16, 1);
-
- addNodesToBucketDB(bid, "0=3,1=2,2=3");
-
- setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:d"));
-
- EXPECT_EQ(
- std::string("BucketId(0x4000000000000001) : "
- "node(idx=0,crc=0x3,docs=3/3,bytes=3/3,trusted=true,active=false,ready=false), "
- "node(idx=2,crc=0x3,docs=3/3,bytes=3/3,trusted=true,active=false,ready=false)"),
- dumpBucket(bid));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, initializing_while_recheck) {
- lib::ClusterState systemState("distributor:1 storage:2 .0.s:i .0.i:0.1");
- setSystemState(systemState);
-
- ASSERT_EQ(messageCount(2), _sender.commands().size());
- ASSERT_EQ(size_t(0), _senderDown.commands().size());
-
- getBucketDBUpdater().recheckBucketInfo(1, makeDocumentBucket(document::BucketId(16, 3)));
-
- for (uint32_t i = 0; i < messageCount(2); ++i) {
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(systemState, *_sender.command(i), 100));
- }
-
- // Now we can pass on system state.
- ASSERT_EQ(size_t(1), _senderDown.commands().size());
- EXPECT_EQ(MessageType::SETSYSTEMSTATE, _senderDown.command(0)->getType());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, bit_change) {
- std::vector<document::BucketId> bucketlist;
-
- {
- setSystemState(lib::ClusterState("bits:14 storage:1 distributor:2"));
-
- ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size());
-
- for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) {
- ASSERT_EQ(_sender.command(bsi)->getType(), MessageType::REQUESTBUCKETINFO);
- const auto &req = dynamic_cast<const RequestBucketInfoCommand &>(*_sender.command(bsi));
- auto sreply = std::make_shared<RequestBucketInfoReply>(req);
- sreply->setAddress(storageAddress(0));
- auto& vec = sreply->getBucketInfo();
- if (req.getBucketSpace() == FixedBucketSpaces::default_space()) {
- int cnt=0;
- for (int i=0; cnt < 2; i++) {
- lib::Distribution distribution = defaultDistributorBucketSpace().getDistribution();
- std::vector<uint16_t> distributors;
- if (distribution.getIdealDistributorNode(
- lib::ClusterState("bits:14 storage:1 distributor:2"),
- document::BucketId(16, i))
- == 0)
- {
- vec.push_back(api::RequestBucketInfoReply::Entry(
- document::BucketId(16, i),
- api::BucketInfo(10,1,1)));
-
- bucketlist.push_back(document::BucketId(16, i));
- cnt++;
- }
- }
- }
-
- getBucketDBUpdater().onRequestBucketInfoReply(sreply);
- }
- }
-
- EXPECT_EQ(std::string("BucketId(0x4000000000000001) : "
- "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"),
- dumpBucket(bucketlist[0]));
- EXPECT_EQ(std::string("BucketId(0x4000000000000002) : "
- "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"),
- dumpBucket(bucketlist[1]));
-
- {
- _sender.clear();
- setSystemState(lib::ClusterState("bits:16 storage:1 distributor:2"));
-
- ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size());
- for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) {
-
- ASSERT_EQ(_sender.command(bsi)->getType(), MessageType::REQUESTBUCKETINFO);
- const auto &req = dynamic_cast<const RequestBucketInfoCommand &>(*_sender.command(bsi));
- auto sreply = std::make_shared<RequestBucketInfoReply>(req);
- sreply->setAddress(storageAddress(0));
- sreply->setResult(api::ReturnCode::OK);
- if (req.getBucketSpace() == FixedBucketSpaces::default_space()) {
- api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo();
-
- for (uint32_t i = 0; i < 3; ++i) {
- vec.push_back(api::RequestBucketInfoReply::Entry(
- document::BucketId(16, i),
- api::BucketInfo(10,1,1)));
- }
-
- vec.push_back(api::RequestBucketInfoReply::Entry(
- document::BucketId(16, 4),
- api::BucketInfo(10,1,1)));
- }
-
- getBucketDBUpdater().onRequestBucketInfoReply(sreply);
- }
- }
-
- EXPECT_EQ(std::string("BucketId(0x4000000000000000) : "
- "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"),
- dumpBucket(document::BucketId(16, 0)));
- EXPECT_EQ(std::string("BucketId(0x4000000000000001) : "
- "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"),
- dumpBucket(document::BucketId(16, 1)));
- EXPECT_EQ(std::string("BucketId(0x4000000000000002) : "
- "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"),
- dumpBucket(document::BucketId(16, 2)));
- EXPECT_EQ(std::string("BucketId(0x4000000000000004) : "
- "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"),
- dumpBucket(document::BucketId(16, 4)));
-
- {
- _sender.clear();
- setSystemState(lib::ClusterState("storage:1 distributor:2 .1.s:i"));
- }
-
- {
- _sender.clear();
- setSystemState(lib::ClusterState("storage:1 distributor:2"));
- }
-};
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, recheck_node_with_failure) {
- ASSERT_NO_FATAL_FAILURE(initializeNodesAndBuckets(3, 5));
-
- _sender.clear();
-
- getBucketDBUpdater().recheckBucketInfo(1, makeDocumentBucket(document::BucketId(16, 3)));
-
- ASSERT_EQ(size_t(1), _sender.commands().size());
-
- uint16_t index = 0;
- {
- auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(0));
- ASSERT_EQ(size_t(1), rbi.getBuckets().size());
- EXPECT_EQ(document::BucketId(16, 3), rbi.getBuckets()[0]);
- auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi);
- const api::StorageMessageAddress *address = _sender.command(0)->getAddress();
- index = address->getIndex();
- reply->setResult(api::ReturnCode::NOT_CONNECTED);
- getBucketDBUpdater().onRequestBucketInfoReply(reply);
- // Trigger that delayed message is sent
- getClock().addSecondsToTime(10);
- getBucketDBUpdater().resendDelayedMessages();
- }
-
- ASSERT_EQ(size_t(2), _sender.commands().size());
-
- setSystemState(
- lib::ClusterState(vespalib::make_string("distributor:1 storage:3 .%d.s:d", index)));
-
- // Recheck bucket.
- {
- auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(1));
- ASSERT_EQ(size_t(1), rbi.getBuckets().size());
- EXPECT_EQ(document::BucketId(16, 3), rbi.getBuckets()[0]);
- auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi);
- reply->setResult(api::ReturnCode::NOT_CONNECTED);
- getBucketDBUpdater().onRequestBucketInfoReply(reply);
- }
-
- // Should not retry since node is down.
- EXPECT_EQ(size_t(2), _sender.commands().size());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, recheck_node) {
- ASSERT_NO_FATAL_FAILURE(initializeNodesAndBuckets(3, 5));
-
- _sender.clear();
-
- getBucketDBUpdater().recheckBucketInfo(1, makeDocumentBucket(document::BucketId(16, 3)));
-
- ASSERT_EQ(size_t(1), _sender.commands().size());
-
- auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(0));
- ASSERT_EQ(size_t(1), rbi.getBuckets().size());
- EXPECT_EQ(document::BucketId(16, 3), rbi.getBuckets()[0]);
-
- auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi);
- reply->getBucketInfo().push_back(
- api::RequestBucketInfoReply::Entry(document::BucketId(16, 3),
- api::BucketInfo(20, 10, 12, 50, 60, true, true)));
- getBucketDBUpdater().onRequestBucketInfoReply(reply);
-
- lib::ClusterState state("distributor:1 storage:3");
- for (uint32_t i = 0; i < 3; i++) {
- EXPECT_EQ(getIdealStr(document::BucketId(16, i), state),
- getNodes(document::BucketId(16, i)));
- }
-
- for (uint32_t i = 4; i < 5; i++) {
- EXPECT_EQ(getIdealStr(document::BucketId(16, i), state),
- getNodes(document::BucketId(16, i)));
- }
-
- BucketDatabase::Entry entry = getBucketDatabase().get(document::BucketId(16, 3));
- ASSERT_TRUE(entry.valid());
-
- const BucketCopy* copy = entry->getNode(1);
- ASSERT_TRUE(copy != nullptr);
- EXPECT_EQ(api::BucketInfo(20,10,12, 50, 60, true, true), copy->getBucketInfo());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, notify_bucket_change) {
- enableDistributorClusterState("distributor:1 storage:1");
-
- addNodesToBucketDB(document::BucketId(16, 1), "0=1234");
- _sender.replies().clear();
-
- {
- api::BucketInfo info(1, 2, 3, 4, 5, true, true);
- auto cmd(std::make_shared<api::NotifyBucketChangeCommand>(
- makeDocumentBucket(document::BucketId(16, 1)), info));
- cmd->setSourceIndex(0);
- getBucketDBUpdater().onNotifyBucketChange(cmd);
- }
-
- {
- api::BucketInfo info(10, 11, 12, 13, 14, false, false);
- auto cmd(std::make_shared<api::NotifyBucketChangeCommand>(
- makeDocumentBucket(document::BucketId(16, 2)), info));
- cmd->setSourceIndex(0);
- getBucketDBUpdater().onNotifyBucketChange(cmd);
- }
-
- // Must receive reply
- ASSERT_EQ(size_t(2), _sender.replies().size());
-
- for (int i = 0; i < 2; ++i) {
- ASSERT_EQ(MessageType::NOTIFYBUCKETCHANGE_REPLY,
- _sender.reply(i)->getType());
- }
-
- // No database update until request bucket info replies have been received.
- EXPECT_EQ(std::string("BucketId(0x4000000000000001) : "
- "node(idx=0,crc=0x4d2,docs=1234/1234,bytes=1234/1234,"
- "trusted=false,active=false,ready=false)"),
- dumpBucket(document::BucketId(16, 1)));
- EXPECT_EQ(std::string("NONEXISTING"), dumpBucket(document::BucketId(16, 2)));
-
- ASSERT_EQ(size_t(2), _sender.commands().size());
-
- std::vector<api::BucketInfo> infos;
- infos.push_back(api::BucketInfo(4567, 200, 2000, 400, 4000, true, true));
- infos.push_back(api::BucketInfo(8999, 300, 3000, 500, 5000, false, false));
-
- for (int i = 0; i < 2; ++i) {
- auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(i));
- ASSERT_EQ(size_t(1), rbi.getBuckets().size());
- EXPECT_EQ(document::BucketId(16, i + 1), rbi.getBuckets()[0]);
-
- auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi);
- reply->getBucketInfo().push_back(
- api::RequestBucketInfoReply::Entry(document::BucketId(16, i + 1),
- infos[i]));
- getBucketDBUpdater().onRequestBucketInfoReply(reply);
- }
-
- EXPECT_EQ(std::string("BucketId(0x4000000000000001) : "
- "node(idx=0,crc=0x11d7,docs=200/400,bytes=2000/4000,trusted=true,active=true,ready=true)"),
- dumpBucket(document::BucketId(16, 1)));
- EXPECT_EQ(std::string("BucketId(0x4000000000000002) : "
- "node(idx=0,crc=0x2327,docs=300/500,bytes=3000/5000,trusted=true,active=false,ready=false)"),
- dumpBucket(document::BucketId(16, 2)));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, notify_bucket_change_from_node_down) {
- enableDistributorClusterState("distributor:1 storage:2");
-
- addNodesToBucketDB(document::BucketId(16, 1), "1=1234");
-
- _sender.replies().clear();
-
- {
- api::BucketInfo info(8999, 300, 3000, 500, 5000, false, false);
- auto cmd(std::make_shared<api::NotifyBucketChangeCommand>(
- makeDocumentBucket(document::BucketId(16, 1)), info));
- cmd->setSourceIndex(0);
- getBucketDBUpdater().onNotifyBucketChange(cmd);
- }
- // Enable here to avoid having request bucket info be silently swallowed
- // (send_request_bucket_info drops message if node is down).
- enableDistributorClusterState("distributor:1 storage:2 .0.s:d");
-
- ASSERT_EQ(std::string("BucketId(0x4000000000000001) : "
- "node(idx=1,crc=0x4d2,docs=1234/1234,bytes=1234/1234,trusted=false,active=false,ready=false)"),
- dumpBucket(document::BucketId(16, 1)));
-
- ASSERT_EQ(size_t(1), _sender.replies().size());
- ASSERT_EQ(MessageType::NOTIFYBUCKETCHANGE_REPLY, _sender.reply(0)->getType());
-
- // Currently, this pending operation will be auto-flushed when the cluster state
- // changes so the behavior is still correct. Keep this test around to prevent
- // regressions here.
- ASSERT_EQ(size_t(1), _sender.commands().size());
- auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(0));
- ASSERT_EQ(size_t(1), rbi.getBuckets().size());
- EXPECT_EQ(document::BucketId(16, 1), rbi.getBuckets()[0]);
-
- auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi);
- reply->getBucketInfo().push_back(
- api::RequestBucketInfoReply::Entry(
- document::BucketId(16, 1),
- api::BucketInfo(8999, 300, 3000, 500, 5000, false, false)));
- getBucketDBUpdater().onRequestBucketInfoReply(reply);
-
- // No change
- EXPECT_EQ(std::string("BucketId(0x4000000000000001) : "
- "node(idx=1,crc=0x4d2,docs=1234/1234,bytes=1234/1234,trusted=false,active=false,ready=false)"),
- dumpBucket(document::BucketId(16, 1)));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-/**
- * Test that NotifyBucketChange received while there's a pending cluster state
- * waits until the cluster state has been enabled as current before it sends off
- * the single bucket info requests. This is to prevent a race condition where
- * the replies to bucket info requests for buckets that would be owned by the
- * distributor in the pending state but not by the current state would be
- * discarded when attempted inserted into the bucket database.
- */
-TEST_F(LegacyBucketDBUpdaterTest, notify_change_with_pending_state_queues_bucket_info_requests) {
- setSystemState(lib::ClusterState("distributor:1 storage:1"));
- ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size());
-
- {
- api::BucketInfo info(8999, 300, 3000, 500, 5000, false, false);
- auto cmd(std::make_shared<api::NotifyBucketChangeCommand>(
- makeDocumentBucket(document::BucketId(16, 1)), info));
- cmd->setSourceIndex(0);
- getBucketDBUpdater().onNotifyBucketChange(cmd);
- }
-
- ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size());
-
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:1 storage:1"),
- _bucketSpaces.size(), 10));
-
- ASSERT_EQ(_bucketSpaces.size() + 1, _sender.commands().size());
-
- {
- auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(_bucketSpaces.size()));
- ASSERT_EQ(size_t(1), rbi.getBuckets().size());
- EXPECT_EQ(document::BucketId(16, 1), rbi.getBuckets()[0]);
- }
- _sender.clear();
-
- // Queue must be cleared once pending state is enabled.
- {
- lib::ClusterState state("distributor:1 storage:2");
- uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1;
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(state, expectedMsgs, dummyBucketsToReturn));
- }
- ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size());
- {
- auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(0));
- EXPECT_EQ(size_t(0), rbi.getBuckets().size());
- }
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, merge_reply) {
- enableDistributorClusterState("distributor:1 storage:3");
-
- addNodesToBucketDB(document::BucketId(16, 1234),
- "0=1234,1=1234,2=1234");
-
- std::vector<api::MergeBucketCommand::Node> nodes;
- nodes.push_back(api::MergeBucketCommand::Node(0));
- nodes.push_back(api::MergeBucketCommand::Node(1));
- nodes.push_back(api::MergeBucketCommand::Node(2));
-
- api::MergeBucketCommand cmd(makeDocumentBucket(document::BucketId(16, 1234)), nodes, 0);
-
- auto reply = std::make_shared<api::MergeBucketReply>(cmd);
-
- _sender.clear();
- getBucketDBUpdater().onMergeBucketReply(reply);
-
- ASSERT_EQ(size_t(3), _sender.commands().size());
-
- for (uint32_t i = 0; i < 3; i++) {
- auto req = std::dynamic_pointer_cast<api::RequestBucketInfoCommand>(_sender.command(i));
-
- ASSERT_TRUE(req.get() != nullptr);
- ASSERT_EQ(size_t(1), req->getBuckets().size());
- EXPECT_EQ(document::BucketId(16, 1234), req->getBuckets()[0]);
-
- auto reqreply = std::make_shared<api::RequestBucketInfoReply>(*req);
- reqreply->getBucketInfo().push_back(
- api::RequestBucketInfoReply::Entry(document::BucketId(16, 1234),
- api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1))));
-
- getBucketDBUpdater().onRequestBucketInfoReply(reqreply);
- }
-
- EXPECT_EQ(std::string("BucketId(0x40000000000004d2) : "
- "node(idx=0,crc=0xa,docs=100/100,bytes=1000/1000,trusted=false,active=false,ready=false), "
- "node(idx=1,crc=0x14,docs=200/200,bytes=2000/2000,trusted=false,active=false,ready=false), "
- "node(idx=2,crc=0x1e,docs=300/300,bytes=3000/3000,trusted=false,active=false,ready=false)"),
- dumpBucket(document::BucketId(16, 1234)));
-};
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, merge_reply_node_down) {
- enableDistributorClusterState("distributor:1 storage:3");
- std::vector<api::MergeBucketCommand::Node> nodes;
-
- addNodesToBucketDB(document::BucketId(16, 1234), "0=1234,1=1234,2=1234");
-
- for (uint32_t i = 0; i < 3; ++i) {
- nodes.push_back(api::MergeBucketCommand::Node(i));
- }
-
- api::MergeBucketCommand cmd(makeDocumentBucket(document::BucketId(16, 1234)), nodes, 0);
-
- auto reply = std::make_shared<api::MergeBucketReply>(cmd);
-
- setSystemState(lib::ClusterState("distributor:1 storage:2"));
-
- _sender.clear();
- getBucketDBUpdater().onMergeBucketReply(reply);
-
- ASSERT_EQ(size_t(2), _sender.commands().size());
-
- for (uint32_t i = 0; i < 2; i++) {
- auto req = std::dynamic_pointer_cast<api::RequestBucketInfoCommand>(_sender.command(i));
-
- ASSERT_TRUE(req.get() != nullptr);
- ASSERT_EQ(size_t(1), req->getBuckets().size());
- EXPECT_EQ(document::BucketId(16, 1234), req->getBuckets()[0]);
-
- auto reqreply = std::make_shared<api::RequestBucketInfoReply>(*req);
- reqreply->getBucketInfo().push_back(
- api::RequestBucketInfoReply::Entry(
- document::BucketId(16, 1234),
- api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1))));
- getBucketDBUpdater().onRequestBucketInfoReply(reqreply);
- }
-
- EXPECT_EQ(std::string("BucketId(0x40000000000004d2) : "
- "node(idx=0,crc=0xa,docs=100/100,bytes=1000/1000,trusted=false,active=false,ready=false), "
- "node(idx=1,crc=0x14,docs=200/200,bytes=2000/2000,trusted=false,active=false,ready=false)"),
- dumpBucket(document::BucketId(16, 1234)));
-};
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, merge_reply_node_down_after_request_sent) {
- enableDistributorClusterState("distributor:1 storage:3");
- std::vector<api::MergeBucketCommand::Node> nodes;
-
- addNodesToBucketDB(document::BucketId(16, 1234), "0=1234,1=1234,2=1234");
-
- for (uint32_t i = 0; i < 3; ++i) {
- nodes.push_back(api::MergeBucketCommand::Node(i));
- }
-
- api::MergeBucketCommand cmd(makeDocumentBucket(document::BucketId(16, 1234)), nodes, 0);
-
- auto reply(std::make_shared<api::MergeBucketReply>(cmd));
-
- _sender.clear();
- getBucketDBUpdater().onMergeBucketReply(reply);
-
- ASSERT_EQ(size_t(3), _sender.commands().size());
-
- setSystemState(lib::ClusterState("distributor:1 storage:2"));
-
- for (uint32_t i = 0; i < 3; i++) {
- auto req = std::dynamic_pointer_cast<api::RequestBucketInfoCommand>(_sender.command(i));
-
- ASSERT_TRUE(req.get() != nullptr);
- ASSERT_EQ(size_t(1), req->getBuckets().size());
- EXPECT_EQ(document::BucketId(16, 1234), req->getBuckets()[0]);
-
- auto reqreply = std::make_shared<api::RequestBucketInfoReply>(*req);
- reqreply->getBucketInfo().push_back(
- api::RequestBucketInfoReply::Entry(
- document::BucketId(16, 1234),
- api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1))));
- getBucketDBUpdater().onRequestBucketInfoReply(reqreply);
- }
-
- EXPECT_EQ(std::string("BucketId(0x40000000000004d2) : "
- "node(idx=0,crc=0xa,docs=100/100,bytes=1000/1000,trusted=false,active=false,ready=false), "
- "node(idx=1,crc=0x14,docs=200/200,bytes=2000/2000,trusted=false,active=false,ready=false)"),
- dumpBucket(document::BucketId(16, 1234)));
-};
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, flush) {
- enableDistributorClusterState("distributor:1 storage:3");
- _sender.clear();
-
- addNodesToBucketDB(document::BucketId(16, 1234), "0=1234,1=1234,2=1234");
-
- std::vector<api::MergeBucketCommand::Node> nodes;
- for (uint32_t i = 0; i < 3; ++i) {
- nodes.push_back(api::MergeBucketCommand::Node(i));
- }
-
- api::MergeBucketCommand cmd(makeDocumentBucket(document::BucketId(16, 1234)), nodes, 0);
-
- auto reply(std::make_shared<api::MergeBucketReply>(cmd));
-
- _sender.clear();
- getBucketDBUpdater().onMergeBucketReply(reply);
-
- ASSERT_EQ(size_t(3), _sender.commands().size());
- ASSERT_EQ(size_t(0), _senderDown.replies().size());
-
- getBucketDBUpdater().flush();
- // Flushing should drop all merge bucket replies
- EXPECT_EQ(size_t(0), _senderDown.commands().size());
-}
-
-std::string
-LegacyBucketDBUpdaterTest::getSentNodes(
- const std::string& oldClusterState,
- const std::string& newClusterState)
-{
- auto fixture = createPendingStateFixtureForStateChange(
- oldClusterState, newClusterState);
-
- sortSentMessagesByIndex(fixture->sender);
-
- std::ostringstream ost;
- for (uint32_t i = 0; i < fixture->sender.commands().size(); i++) {
- auto& req = dynamic_cast<RequestBucketInfoCommand&>(*fixture->sender.command(i));
-
- if (i > 0) {
- ost << ",";
- }
-
- ost << req.getAddress()->getIndex();
- }
-
- return ost.str();
-}
-
-std::string
-LegacyBucketDBUpdaterTest::getSentNodesDistributionChanged(
- const std::string& oldClusterState)
-{
- DistributorMessageSenderStub sender;
-
- framework::defaultimplementation::FakeClock clock;
- ClusterInformation::CSP clusterInfo(createClusterInfo(oldClusterState));
- std::unique_ptr<PendingClusterState> state(
- PendingClusterState::createForDistributionChange(
- clock, clusterInfo, sender, getBucketSpaceRepo(), api::Timestamp(1)));
-
- sortSentMessagesByIndex(sender);
-
- std::ostringstream ost;
- for (uint32_t i = 0; i < sender.commands().size(); i++) {
- auto& req = dynamic_cast<RequestBucketInfoCommand&>(*sender.command(i));
-
- if (i > 0) {
- ost << ",";
- }
-
- ost << req.getAddress()->getIndex();
- }
-
- return ost.str();
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_send_messages) {
- EXPECT_EQ(getNodeList({0, 1, 2}),
- getSentNodes("cluster:d",
- "distributor:1 storage:3"));
-
- EXPECT_EQ(getNodeList({0, 1}),
- getSentNodes("cluster:d",
- "distributor:1 storage:3 .2.s:m"));
-
- EXPECT_EQ(getNodeList({2}),
- getSentNodes("distributor:1 storage:2",
- "distributor:1 storage:3"));
-
- EXPECT_EQ(getNodeList({2, 3, 4, 5}),
- getSentNodes("distributor:1 storage:2",
- "distributor:1 storage:6"));
-
- EXPECT_EQ(getNodeList({0, 1, 2}),
- getSentNodes("distributor:4 storage:3",
- "distributor:3 storage:3"));
-
- EXPECT_EQ(getNodeList({0, 1, 2, 3}),
- getSentNodes("distributor:4 storage:3",
- "distributor:4 .2.s:d storage:4"));
-
- EXPECT_EQ(std::string(""),
- getSentNodes("distributor:4 storage:3",
- "distributor:4 .0.s:d storage:4"));
-
- EXPECT_EQ(std::string(""),
- getSentNodes("distributor:3 storage:3",
- "distributor:4 storage:3"));
-
- EXPECT_EQ(getNodeList({2}),
- getSentNodes("distributor:3 storage:3 .2.s:i",
- "distributor:3 storage:3"));
-
- EXPECT_EQ(getNodeList({1}),
- getSentNodes("distributor:3 storage:3 .1.s:d",
- "distributor:3 storage:3"));
-
- EXPECT_EQ(getNodeList({1, 2, 4}),
- getSentNodes("distributor:3 storage:4 .1.s:d .2.s:i",
- "distributor:3 storage:5"));
-
- EXPECT_EQ(std::string(""),
- getSentNodes("distributor:1 storage:3",
- "cluster:d"));
-
- EXPECT_EQ(std::string(""),
- getSentNodes("distributor:1 storage:3",
- "distributor:1 storage:3"));
-
- EXPECT_EQ(std::string(""),
- getSentNodes("distributor:1 storage:3",
- "cluster:d distributor:1 storage:6"));
-
- EXPECT_EQ(std::string(""),
- getSentNodes("distributor:3 storage:3",
- "distributor:3 .2.s:m storage:3"));
-
- EXPECT_EQ(getNodeList({0, 1, 2}),
- getSentNodes("distributor:3 .2.s:m storage:3",
- "distributor:3 .2.s:d storage:3"));
-
- EXPECT_EQ(std::string(""),
- getSentNodes("distributor:3 .2.s:m storage:3",
- "distributor:3 storage:3"));
-
- EXPECT_EQ(getNodeList({0, 1, 2}),
- getSentNodesDistributionChanged("distributor:3 storage:3"));
-
- EXPECT_EQ(getNodeList({0, 1}),
- getSentNodes("distributor:10 storage:2",
- "distributor:10 .1.s:d storage:2"));
-
- EXPECT_EQ(std::string(""),
- getSentNodes("distributor:2 storage:2",
- "distributor:3 .2.s:i storage:2"));
-
- EXPECT_EQ(getNodeList({0, 1, 2}),
- getSentNodes("distributor:3 storage:3",
- "distributor:3 .2.s:s storage:3"));
-
- EXPECT_EQ(std::string(""),
- getSentNodes("distributor:3 .2.s:s storage:3",
- "distributor:3 .2.s:d storage:3"));
-
- EXPECT_EQ(getNodeList({1}),
- getSentNodes("distributor:3 storage:3 .1.s:m",
- "distributor:3 storage:3"));
-
- EXPECT_EQ(std::string(""),
- getSentNodes("distributor:3 storage:3",
- "distributor:3 storage:3 .1.s:m"));
-};
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_receive) {
- DistributorMessageSenderStub sender;
-
- auto cmd(std::make_shared<api::SetSystemStateCommand>(
- lib::ClusterState("distributor:1 storage:3")));
-
- framework::defaultimplementation::FakeClock clock;
- ClusterInformation::CSP clusterInfo(createClusterInfo("cluster:d"));
- OutdatedNodesMap outdatedNodesMap;
- std::unique_ptr<PendingClusterState> state(
- PendingClusterState::createForClusterStateChange(
- clock, clusterInfo, sender, getBucketSpaceRepo(),
- cmd, outdatedNodesMap, api::Timestamp(1)));
-
- ASSERT_EQ(messageCount(3), sender.commands().size());
-
- sortSentMessagesByIndex(sender);
-
- std::ostringstream ost;
- for (uint32_t i = 0; i < sender.commands().size(); i++) {
- auto* req = dynamic_cast<RequestBucketInfoCommand*>(sender.command(i).get());
- ASSERT_TRUE(req != nullptr);
-
- auto rep = std::make_shared<RequestBucketInfoReply>(*req);
-
- rep->getBucketInfo().push_back(
- RequestBucketInfoReply::Entry(
- document::BucketId(16, i),
- api::BucketInfo(i, i, i, i, i)));
-
- ASSERT_TRUE(state->onRequestBucketInfoReply(rep));
- ASSERT_EQ((i == (sender.commands().size() - 1)), state->done());
- }
-
- auto& pendingTransition = state->getPendingBucketSpaceDbTransition(makeBucketSpace());
- EXPECT_EQ(3, (int)pendingTransition.results().size());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_with_group_down) {
- std::string config(getDistConfig6Nodes4Groups());
- config += "distributor_auto_ownership_transfer_on_whole_group_down true\n";
- setDistribution(config);
-
- // Group config has nodes {0, 1}, {2, 3}, {4, 5}
- // We're node index 0.
-
- // Entire group 1 goes down. Must refetch from all nodes.
- EXPECT_EQ(getNodeList({0, 1, 2, 3, 4, 5}),
- getSentNodes("distributor:6 storage:6",
- "distributor:6 .2.s:d .3.s:d storage:6"));
-
- // But don't fetch if not the entire group is down.
- EXPECT_EQ(std::string(""),
- getSentNodes("distributor:6 storage:6",
- "distributor:6 .2.s:d storage:6"));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_with_group_down_and_no_handover) {
- std::string config(getDistConfig6Nodes4Groups());
- config += "distributor_auto_ownership_transfer_on_whole_group_down false\n";
- setDistribution(config);
-
- // Group is down, but config says to not do anything about it.
- EXPECT_EQ(getNodeList({0, 1, 2, 3, 4, 5}, _bucketSpaces.size() - 1),
- getSentNodes("distributor:6 storage:6",
- "distributor:6 .2.s:d .3.s:d storage:6"));
-}
-
-namespace {
-
-void
-parseInputData(const std::string& data,
- uint64_t timestamp,
- PendingClusterState& state,
- bool includeBucketInfo)
-{
- vespalib::StringTokenizer tokenizer(data, "|");
- for (uint32_t i = 0; i < tokenizer.size(); i++) {
- vespalib::StringTokenizer tok2(tokenizer[i], ":");
-
- uint16_t node = atoi(tok2[0].data());
-
- state.setNodeReplied(node);
- auto& pendingTransition = state.getPendingBucketSpaceDbTransition(makeBucketSpace());
-
- vespalib::StringTokenizer tok3(tok2[1], ",");
- for (uint32_t j = 0; j < tok3.size(); j++) {
- if (includeBucketInfo) {
- vespalib::StringTokenizer tok4(tok3[j], "/");
-
- pendingTransition.addNodeInfo(
- document::BucketId(16, atoi(tok4[0].data())),
- BucketCopy(
- timestamp,
- node,
- api::BucketInfo(
- atoi(tok4[1].data()),
- atoi(tok4[2].data()),
- atoi(tok4[3].data()),
- atoi(tok4[2].data()),
- atoi(tok4[3].data()))));
- } else {
- pendingTransition.addNodeInfo(
- document::BucketId(16, atoi(tok3[j].data())),
- BucketCopy(timestamp,
- node,
- api::BucketInfo(3, 3, 3, 3, 3)));
- }
- }
- }
-}
-
-struct BucketDumper : public BucketDatabase::EntryProcessor
-{
- std::ostringstream ost;
- bool _includeBucketInfo;
-
- explicit BucketDumper(bool includeBucketInfo)
- : _includeBucketInfo(includeBucketInfo)
- {
- }
-
- bool process(const BucketDatabase::ConstEntryRef& e) override {
- document::BucketId bucketId(e.getBucketId());
-
- ost << (uint32_t)bucketId.getRawId() << ":";
- for (uint32_t i = 0; i < e->getNodeCount(); ++i) {
- if (i > 0) {
- ost << ",";
- }
- const BucketCopy& copy(e->getNodeRef(i));
- ost << copy.getNode();
- if (_includeBucketInfo) {
- ost << "/" << copy.getChecksum()
- << "/" << copy.getDocumentCount()
- << "/" << copy.getTotalDocumentSize()
- << "/" << (copy.trusted() ? "t" : "u");
- }
- }
- ost << "|";
- return true;
- }
-};
-
-}
-
-std::string
-LegacyBucketDBUpdaterTest::mergeBucketLists(
- const lib::ClusterState& oldState,
- const std::string& existingData,
- const lib::ClusterState& newState,
- const std::string& newData,
- bool includeBucketInfo)
-{
- framework::defaultimplementation::FakeClock clock;
- framework::MilliSecTimer timer(clock);
-
- DistributorMessageSenderStub sender;
- OutdatedNodesMap outdatedNodesMap;
-
- {
- auto cmd(std::make_shared<api::SetSystemStateCommand>(oldState));
-
- api::Timestamp beforeTime(1);
-
- ClusterInformation::CSP clusterInfo(createClusterInfo("cluster:d"));
- std::unique_ptr<PendingClusterState> state(
- PendingClusterState::createForClusterStateChange(
- clock, clusterInfo, sender, getBucketSpaceRepo(),
- cmd, outdatedNodesMap, beforeTime));
-
- parseInputData(existingData, beforeTime, *state, includeBucketInfo);
- state->mergeIntoBucketDatabases();
- }
-
- BucketDumper dumper_tmp(true);
- getBucketDatabase().forEach(dumper_tmp);
-
- {
- auto cmd(std::make_shared<api::SetSystemStateCommand>(
- lib::ClusterState(newState)));
-
- api::Timestamp afterTime(2);
-
- ClusterInformation::CSP clusterInfo(createClusterInfo(oldState.toString()));
- std::unique_ptr<PendingClusterState> state(
- PendingClusterState::createForClusterStateChange(
- clock, clusterInfo, sender, getBucketSpaceRepo(),
- cmd, outdatedNodesMap, afterTime));
-
- parseInputData(newData, afterTime, *state, includeBucketInfo);
- state->mergeIntoBucketDatabases();
- }
-
- BucketDumper dumper(includeBucketInfo);
- auto &bucketDb(defaultDistributorBucketSpace().getBucketDatabase());
- bucketDb.forEach(dumper);
- bucketDb.clear();
- return dumper.ost.str();
-}
-
-std::string
-LegacyBucketDBUpdaterTest::mergeBucketLists(const std::string& existingData,
- const std::string& newData,
- bool includeBucketInfo)
-{
- return mergeBucketLists(
- lib::ClusterState("distributor:1 storage:3"),
- existingData,
- lib::ClusterState("distributor:1 storage:3"),
- newData,
- includeBucketInfo);
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_merge) {
- // Simple initializing case - ask all nodes for info
- EXPECT_EQ(
- // Result is on the form: [bucket w/o count bits]:[node indexes]|..
- std::string("4:0,1|2:0,1|6:1,2|1:0,2|5:2,0|3:2,1|"),
- // Input is on the form: [node]:[bucket w/o count bits]|...
- mergeBucketLists(
- "",
- "0:1,2,4,5|1:2,3,4,6|2:1,3,5,6"));
-
- // New node came up
- EXPECT_EQ(
- std::string("4:0,1|2:0,1|6:1,2,3|1:0,2,3|5:2,0,3|3:2,1,3|"),
- mergeBucketLists(
- "0:1,2,4,5|1:2,3,4,6|2:1,3,5,6",
- "3:1,3,5,6"));
-
- // Node came up with some buckets removed and some added
- // Buckets that were removed should not be removed as the node
- // didn't lose a disk.
- EXPECT_EQ(
- std::string("8:0|4:0,1|2:0,1|6:1,0,2|1:0,2|5:2,0|3:2,1|"),
- mergeBucketLists(
- "0:1,2,4,5|1:2,3,4,6|2:1,3,5,6",
- "0:1,2,6,8"));
-
- // Bucket info format is "bucketid/checksum/count/size"
- // Node went from initializing to up and invalid bucket went to empty.
- EXPECT_EQ(
- std::string("2:0/0/0/0/t|"),
- mergeBucketLists(
- "0:2/0/0/1",
- "0:2/0/0/0",
- true));
-
- EXPECT_EQ(std::string("5:1/2/3/4/u,0/0/0/0/u|"),
- mergeBucketLists("", "0:5/0/0/0|1:5/2/3/4", true));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_merge_replica_changed) {
- // Node went from initializing to up and non-invalid bucket changed.
- EXPECT_EQ(
- std::string("2:0/2/3/4/t|3:0/2/4/6/t|"),
- mergeBucketLists(
- lib::ClusterState("distributor:1 storage:1 .0.s:i"),
- "0:2/1/2/3,3/2/4/6",
- lib::ClusterState("distributor:1 storage:1"),
- "0:2/2/3/4,3/2/4/6",
- true));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, no_db_resurrection_for_bucket_not_owned_in_current_state) {
- document::BucketId bucket(16, 3);
- lib::ClusterState stateBefore("distributor:1 storage:1");
- {
- uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1;
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn));
- }
- _sender.clear();
-
- getBucketDBUpdater().recheckBucketInfo(0, makeDocumentBucket(bucket));
-
- ASSERT_EQ(size_t(1), _sender.commands().size());
- std::shared_ptr<api::RequestBucketInfoCommand> rbi(
- std::dynamic_pointer_cast<RequestBucketInfoCommand>(
- _sender.command(0)));
-
- lib::ClusterState stateAfter("distributor:3 storage:3");
-
- {
- uint32_t expectedMsgs = messageCount(2), dummyBucketsToReturn = 1;
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateAfter, expectedMsgs, dummyBucketsToReturn));
- }
- EXPECT_FALSE(getDistributorBucketSpace().get_bucket_ownership_flags(bucket).owned_in_current_state());
-
- ASSERT_NO_FATAL_FAILURE(sendFakeReplyForSingleBucketRequest(*rbi));
-
- EXPECT_EQ(std::string("NONEXISTING"), dumpBucket(bucket));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, no_db_resurrection_for_bucket_not_owned_in_pending_state) {
- document::BucketId bucket(16, 3);
- lib::ClusterState stateBefore("distributor:1 storage:1");
- {
- uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1;
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn));
- }
- _sender.clear();
-
- getBucketDBUpdater().recheckBucketInfo(0, makeDocumentBucket(bucket));
-
- ASSERT_EQ(size_t(1), _sender.commands().size());
- std::shared_ptr<api::RequestBucketInfoCommand> rbi(
- std::dynamic_pointer_cast<RequestBucketInfoCommand>(
- _sender.command(0)));
-
- lib::ClusterState stateAfter("distributor:3 storage:3");
- // Set, but _don't_ enable cluster state. We want it to be pending.
- setSystemState(stateAfter);
- EXPECT_TRUE(getDistributorBucketSpace().get_bucket_ownership_flags(bucket).owned_in_current_state());
- EXPECT_FALSE(getDistributorBucketSpace().get_bucket_ownership_flags(bucket).owned_in_pending_state());
-
- ASSERT_NO_FATAL_FAILURE(sendFakeReplyForSingleBucketRequest(*rbi));
-
- EXPECT_EQ(std::string("NONEXISTING"), dumpBucket(bucket));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-/*
- * If we get a distribution config change, it's important that cluster states that
- * arrive after this--but _before_ the pending cluster state has finished--must trigger
- * a full bucket info fetch no matter what the cluster state change was! Otherwise, we
- * will with a high likelihood end up not getting the complete view of the buckets in
- * the cluster.
- */
-TEST_F(LegacyBucketDBUpdaterTest, cluster_state_always_sends_full_fetch_when_distribution_change_pending) {
- lib::ClusterState stateBefore("distributor:6 storage:6");
- {
- uint32_t expectedMsgs = messageCount(6), dummyBucketsToReturn = 1;
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn));
- }
- _sender.clear();
- std::string distConfig(getDistConfig6Nodes2Groups());
- setDistribution(distConfig);
-
- sortSentMessagesByIndex(_sender);
- ASSERT_EQ(messageCount(6), _sender.commands().size());
- // Suddenly, a wild cluster state change appears! Even though this state
- // does not in itself imply any bucket changes, it will still overwrite the
- // pending cluster state and thus its state of pending bucket info requests.
- setSystemState(lib::ClusterState("distributor:6 .2.t:12345 storage:6"));
-
- ASSERT_EQ(messageCount(12), _sender.commands().size());
-
- // Send replies for first messageCount(6) (outdated requests).
- int numBuckets = 10;
- for (uint32_t i = 0; i < messageCount(6); ++i) {
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:6 storage:6"),
- *_sender.command(i), numBuckets));
- }
- // No change from these.
- ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(1, "distributor:6 storage:6"));
-
- // Send for current pending.
- for (uint32_t i = 0; i < messageCount(6); ++i) {
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:6 .2.t:12345 storage:6"),
- *_sender.command(i + messageCount(6)),
- numBuckets));
- }
- ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(numBuckets, "distributor:6 storage:6"));
- _sender.clear();
-
- // No more pending global fetch; this should be a no-op state.
- setSystemState(lib::ClusterState("distributor:6 .3.t:12345 storage:6"));
- EXPECT_EQ(size_t(0), _sender.commands().size());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, changed_distribution_config_triggers_recovery_mode) {
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), messageCount(6), 20));
- _sender.clear();
- EXPECT_TRUE(distributor_is_in_recovery_mode());
- complete_recovery_mode();
- EXPECT_FALSE(distributor_is_in_recovery_mode());
-
- std::string distConfig(getDistConfig6Nodes4Groups());
- setDistribution(distConfig);
- sortSentMessagesByIndex(_sender);
- // No replies received yet, still no recovery mode.
- EXPECT_FALSE(distributor_is_in_recovery_mode());
-
- ASSERT_EQ(messageCount(6), _sender.commands().size());
- uint32_t numBuckets = 10;
- for (uint32_t i = 0; i < messageCount(6); ++i) {
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:6 storage:6"),
- *_sender.command(i), numBuckets));
- }
-
- // Pending cluster state (i.e. distribution) has been enabled, which should
- // cause recovery mode to be entered.
- EXPECT_TRUE(distributor_is_in_recovery_mode());
- complete_recovery_mode();
- EXPECT_FALSE(distributor_is_in_recovery_mode());
-}
-
-namespace {
-
-template <typename Func>
-struct FunctorProcessor : BucketDatabase::EntryProcessor {
- Func _f;
-
- template <typename F>
- explicit FunctorProcessor(F&& f) : _f(std::forward<F>(f)) {}
-
- bool process(const BucketDatabase::ConstEntryRef& e) override {
- _f(e);
- return true;
- }
-};
-
-template <typename Func>
-std::unique_ptr<BucketDatabase::EntryProcessor> func_processor(Func&& f) {
- return std::make_unique<FunctorProcessor<Func>>(std::forward<Func>(f));
-}
-
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, changed_distribution_config_does_not_elide_bucket_db_pruning) {
- setDistribution(getDistConfig3Nodes1Group());
-
- constexpr uint32_t n_buckets = 100;
- ASSERT_NO_FATAL_FAILURE(
- setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), messageCount(6), n_buckets));
- _sender.clear();
-
- // Config implies a different node set than the current cluster state, so it's crucial that
- // DB pruning is _not_ elided. Yes, this is inherently racing with cluster state changes and
- // should be changed to be atomic and controlled by the cluster controller instead of config.
- // But this is where we currently are.
- setDistribution(getDistConfig6Nodes2Groups());
-
- getBucketDatabase().forEach(*func_processor([&](const auto& e) {
- EXPECT_TRUE(getDistributorBucketSpace().get_bucket_ownership_flags(e.getBucketId()).owned_in_pending_state());
- }));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, newly_added_buckets_have_current_time_as_gc_timestamp) {
- getClock().setAbsoluteTimeInSeconds(101234);
- lib::ClusterState stateBefore("distributor:1 storage:1");
- {
- uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1;
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn));
- }
-
- // setAndEnableClusterState adds n buckets with id (16, i)
- document::BucketId bucket(16, 0);
- BucketDatabase::Entry e(getBucket(bucket));
- ASSERT_TRUE(e.valid());
- EXPECT_EQ(uint32_t(101234), e->getLastGarbageCollectionTime());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, newer_mutations_not_overwritten_by_earlier_bucket_fetch) {
- {
- lib::ClusterState stateBefore("distributor:1 storage:1 .0.s:i");
- uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 0;
- // This step is required to make the distributor ready for accepting
- // the below explicit database insertion towards node 0.
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs,
- dummyBucketsToReturn));
- }
- _sender.clear();
- getClock().setAbsoluteTimeInSeconds(1000);
- lib::ClusterState state("distributor:1 storage:1");
- setSystemState(state);
- ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size());
-
- // Before replying with the bucket info, simulate the arrival of a mutation
- // reply that alters the state of the bucket with information that will be
- // more recent that what is returned by the bucket info. This information
- // must not be lost when the bucket info is later merged into the database.
- document::BucketId bucket(16, 1);
- constexpr uint64_t insertionTimestamp = 1001ULL * 1000000;
- api::BucketInfo wantedInfo(5, 6, 7);
- getBucketDBUpdater().operation_context().update_bucket_database(
- makeDocumentBucket(bucket),
- BucketCopy(insertionTimestamp, 0, wantedInfo),
- DatabaseUpdate::CREATE_IF_NONEXISTING);
-
- getClock().setAbsoluteTimeInSeconds(1002);
- constexpr uint32_t bucketsReturned = 10; // Buckets (16, 0) ... (16, 9)
- // Return bucket information which on the timeline might originate from
- // anywhere between [1000, 1002]. Our assumption is that any mutations
- // taking place after t=1000 must have its reply received and processed
- // by this distributor and timestamped strictly higher than t=1000 (modulo
- // clock skew, of course, but that is outside the scope of this). A mutation
- // happening before t=1000 but receiving a reply at t>1000 does not affect
- // correctness, as this should contain the same bucket info as that
- // contained in the full bucket reply and the DB update is thus idempotent.
- for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) {
- ASSERT_NO_FATAL_FAILURE(fakeBucketReply(state, *_sender.command(i), bucketsReturned));
- }
-
- BucketDatabase::Entry e(getBucket(bucket));
- ASSERT_EQ(uint32_t(1), e->getNodeCount());
- EXPECT_EQ(wantedInfo, e->getNodeRef(0).getBucketInfo());
-}
-
-std::vector<uint16_t>
-LegacyBucketDBUpdaterTest::getSendSet() const
-{
- std::vector<uint16_t> nodes;
- std::transform(_sender.commands().begin(),
- _sender.commands().end(),
- std::back_inserter(nodes),
- [](auto& cmd)
- {
- auto& req(dynamic_cast<const api::RequestBucketInfoCommand&>(*cmd));
- return req.getAddress()->getIndex();
- });
- return nodes;
-}
-
-std::vector<uint16_t>
-LegacyBucketDBUpdaterTest::getSentNodesWithPreemption(
- const std::string& oldClusterState,
- uint32_t expectedOldStateMessages,
- const std::string& preemptedClusterState,
- const std::string& newClusterState)
-{
- lib::ClusterState stateBefore(oldClusterState);
- uint32_t dummyBucketsToReturn = 10;
- // FIXME cannot chain assertion checks in non-void function
- setAndEnableClusterState(lib::ClusterState(oldClusterState),
- expectedOldStateMessages,
- dummyBucketsToReturn);
-
- _sender.clear();
-
- setSystemState(lib::ClusterState(preemptedClusterState));
- _sender.clear();
- // Do not allow the pending state to become the active state; trigger a
- // new transition without ACKing the info requests first. This will
- // overwrite the pending state entirely.
- setSystemState(lib::ClusterState(newClusterState));
- return getSendSet();
-}
-
-using nodeVec = std::vector<uint16_t>;
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-/*
- * If we don't carry over the set of nodes that we need to fetch from,
- * a naive comparison between the active state and the new state will
- * make it appear to the distributor that nothing has changed, as any
- * database modifications caused by intermediate states will not be
- * accounted for (basically the ABA problem in a distributed setting).
- */
-TEST_F(LegacyBucketDBUpdaterTest, preempted_distributor_change_carries_node_set_over_to_next_state_fetch) {
- EXPECT_EQ(
- expandNodeVec({0, 1, 2, 3, 4, 5}),
- getSentNodesWithPreemption("version:1 distributor:6 storage:6",
- messageCount(6),
- "version:2 distributor:6 .5.s:d storage:6",
- "version:3 distributor:6 storage:6"));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, preempted_storage_change_carries_node_set_over_to_next_state_fetch) {
- EXPECT_EQ(
- expandNodeVec({2, 3}),
- getSentNodesWithPreemption(
- "version:1 distributor:6 storage:6 .2.s:d",
- messageCount(5),
- "version:2 distributor:6 storage:6 .2.s:d .3.s:d",
- "version:3 distributor:6 storage:6"));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, preempted_storage_node_down_must_be_re_fetched) {
- EXPECT_EQ(
- expandNodeVec({2}),
- getSentNodesWithPreemption(
- "version:1 distributor:6 storage:6",
- messageCount(6),
- "version:2 distributor:6 storage:6 .2.s:d",
- "version:3 distributor:6 storage:6"));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, do_not_send_to_preempted_node_now_in_down_state) {
- EXPECT_EQ(
- nodeVec{},
- getSentNodesWithPreemption(
- "version:1 distributor:6 storage:6 .2.s:d",
- messageCount(5),
- "version:2 distributor:6 storage:6", // Sends to 2.
- "version:3 distributor:6 storage:6 .2.s:d")); // 2 down again.
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, doNotSendToPreemptedNodeNotPartOfNewState) {
- // Even though 100 nodes are preempted, not all of these should be part
- // of the request afterwards when only 6 are part of the state.
- EXPECT_EQ(
- expandNodeVec({0, 1, 2, 3, 4, 5}),
- getSentNodesWithPreemption(
- "version:1 distributor:6 storage:100",
- messageCount(100),
- "version:2 distributor:5 .4.s:d storage:100",
- "version:3 distributor:6 storage:6"));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, outdated_node_set_cleared_after_successful_state_completion) {
- lib::ClusterState stateBefore(
- "version:1 distributor:6 storage:6 .1.t:1234");
- uint32_t expectedMsgs = messageCount(6), dummyBucketsToReturn = 10;
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn));
- _sender.clear();
- // New cluster state that should not by itself trigger any new fetches,
- // unless outdated node set is somehow not cleared after an enabled
- // (completed) cluster state has been set.
- lib::ClusterState stateAfter("version:3 distributor:6 storage:6");
- setSystemState(stateAfter);
- EXPECT_EQ(size_t(0), _sender.commands().size());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest (despite being disabled)
-// XXX test currently disabled since distribution config currently isn't used
-// at all in order to deduce the set of nodes to send to. This might not matter
-// in practice since it is assumed that the cluster state matching the new
-// distribution config will follow very shortly after the config has been
-// applied to the node. The new cluster state will then send out requests to
-// the correct node set.
-TEST_F(LegacyBucketDBUpdaterTest, DISABLED_cluster_config_downsize_only_sends_to_available_nodes) {
- uint32_t expectedMsgs = 6, dummyBucketsToReturn = 20;
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"),
- expectedMsgs, dummyBucketsToReturn));
- _sender.clear();
-
- // Intentionally trigger a racing config change which arrives before the
- // new cluster state representing it.
- std::string distConfig(getDistConfig3Nodes1Group());
- setDistribution(distConfig);
- sortSentMessagesByIndex(_sender);
-
- EXPECT_EQ((nodeVec{0, 1, 2}), getSendSet());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-/**
- * Test scenario where a cluster is downsized by removing a subset of the nodes
- * from the distribution configuration. The system must be able to deal with
- * a scenario where the set of nodes between two cluster states across a config
- * change may differ.
- *
- * See VESPA-790 for details.
- */
-TEST_F(LegacyBucketDBUpdaterTest, node_missing_from_config_is_treated_as_needing_ownership_transfer) {
- uint32_t expectedMsgs = messageCount(3), dummyBucketsToReturn = 1;
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(lib::ClusterState("distributor:3 storage:3"),
- expectedMsgs, dummyBucketsToReturn));
- _sender.clear();
-
- // Cluster goes from {0, 1, 2} -> {0, 1}. This leaves us with a config
- // that does not contain node 2 while the _active_ cluster state still
- // contains this node.
- const char* downsizeCfg =
- "redundancy 2\n"
- "distributor_auto_ownership_transfer_on_whole_group_down true\n"
- "group[2]\n"
- "group[0].name \"invalid\"\n"
- "group[0].index \"invalid\"\n"
- "group[0].partitions 1|*\n"
- "group[0].nodes[0]\n"
- "group[1].name rack0\n"
- "group[1].index 0\n"
- "group[1].nodes[2]\n"
- "group[1].nodes[0].index 0\n"
- "group[1].nodes[1].index 1\n";
-
- setDistribution(downsizeCfg);
- sortSentMessagesByIndex(_sender);
- _sender.clear();
-
- // Attempt to apply state with {0, 1} set. This will compare the new state
- // with the previous state, which still has node 2.
- expectedMsgs = messageCount(2);
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(lib::ClusterState("distributor:2 storage:2"),
- expectedMsgs, dummyBucketsToReturn));
-
- EXPECT_EQ(expandNodeVec({0, 1}), getSendSet());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, changed_distributor_set_implies_ownership_transfer) {
- auto fixture = createPendingStateFixtureForStateChange(
- "distributor:2 storage:2", "distributor:1 storage:2");
- EXPECT_TRUE(fixture->state->hasBucketOwnershipTransfer());
-
- fixture = createPendingStateFixtureForStateChange(
- "distributor:2 storage:2", "distributor:2 .1.s:d storage:2");
- EXPECT_TRUE(fixture->state->hasBucketOwnershipTransfer());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, unchanged_distributor_set_implies_no_ownership_transfer) {
- auto fixture = createPendingStateFixtureForStateChange(
- "distributor:2 storage:2", "distributor:2 storage:1");
- EXPECT_FALSE(fixture->state->hasBucketOwnershipTransfer());
-
- fixture = createPendingStateFixtureForStateChange(
- "distributor:2 storage:2", "distributor:2 storage:2 .1.s:d");
- EXPECT_FALSE(fixture->state->hasBucketOwnershipTransfer());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, changed_distribution_config_implies_ownership_transfer) {
- auto fixture = createPendingStateFixtureForDistributionChange(
- "distributor:2 storage:2");
- EXPECT_TRUE(fixture->state->hasBucketOwnershipTransfer());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, transition_time_tracked_for_single_state_change) {
- ASSERT_NO_FATAL_FAILURE(completeStateTransitionInSeconds("distributor:2 storage:2", 5, messageCount(2)));
-
- EXPECT_EQ(uint64_t(5000), lastTransitionTimeInMillis());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, transition_time_reset_across_non_preempting_state_changes) {
- ASSERT_NO_FATAL_FAILURE(completeStateTransitionInSeconds("distributor:2 storage:2", 5, messageCount(2)));
- ASSERT_NO_FATAL_FAILURE(completeStateTransitionInSeconds("distributor:2 storage:3", 3, messageCount(1)));
-
- EXPECT_EQ(uint64_t(3000), lastTransitionTimeInMillis());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, transition_time_tracked_for_distribution_config_change) {
- lib::ClusterState state("distributor:2 storage:2");
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(state, messageCount(2), 1));
-
- _sender.clear();
- std::string distConfig(getDistConfig3Nodes1Group());
- setDistribution(distConfig);
- getClock().addSecondsToTime(4);
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(state, messageCount(2)));
- EXPECT_EQ(uint64_t(4000), lastTransitionTimeInMillis());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, transition_time_tracked_across_preempted_transitions) {
- _sender.clear();
- lib::ClusterState state("distributor:2 storage:2");
- setSystemState(state);
- getClock().addSecondsToTime(5);
- // Pre-empted with new state here, which will push out the old pending
- // state and replace it with a new one. We should still count the time
- // used processing the old state.
- ASSERT_NO_FATAL_FAILURE(completeStateTransitionInSeconds("distributor:2 storage:3", 3, messageCount(3)));
-
- EXPECT_EQ(uint64_t(8000), lastTransitionTimeInMillis());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-/*
- * Brief reminder on test DSL for checking bucket merge operations:
- *
- * mergeBucketLists() takes as input strings of the format
- * <node>:<raw bucket id>/<checksum>/<num docs>/<doc size>|<node>:
- * and returns a string describing the bucket DB post-merge with the format
- * <raw bucket id>:<node>/<checksum>/<num docs>/<doc size>,<node>:....|<raw bucket id>:....
- *
- * Yes, the order of node<->bucket id is reversed between the two, perhaps to make sure you're awake.
- */
-
-TEST_F(LegacyBucketDBUpdaterTest, batch_update_of_existing_diverging_replicas_does_not_mark_any_as_trusted) {
- // Replacing bucket information for content node 0 should not mark existing
- // untrusted replica as trusted as a side effect.
- EXPECT_EQ(
- std::string("5:1/7/8/9/u,0/1/2/3/u|"),
- mergeBucketLists(
- lib::ClusterState("distributor:1 storage:3 .0.s:i"),
- "0:5/0/0/0|1:5/7/8/9",
- lib::ClusterState("distributor:1 storage:3 .0.s:u"),
- "0:5/1/2/3|1:5/7/8/9", true));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, batch_add_of_new_diverging_replicas_does_not_mark_any_as_trusted) {
- EXPECT_EQ(std::string("5:1/7/8/9/u,0/1/2/3/u|"),
- mergeBucketLists("", "0:5/1/2/3|1:5/7/8/9", true));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, batch_add_with_single_resulting_replica_implicitly_marks_as_trusted) {
- EXPECT_EQ(std::string("5:0/1/2/3/t|"),
- mergeBucketLists("", "0:5/1/2/3", true));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, identity_update_of_single_replica_does_not_clear_trusted) {
- EXPECT_EQ(std::string("5:0/1/2/3/t|"),
- mergeBucketLists("0:5/1/2/3", "0:5/1/2/3", true));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, identity_update_of_diverging_untrusted_replicas_does_not_mark_any_as_trusted) {
- EXPECT_EQ(std::string("5:1/7/8/9/u,0/1/2/3/u|"),
- mergeBucketLists("0:5/1/2/3|1:5/7/8/9", "0:5/1/2/3|1:5/7/8/9", true));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, adding_diverging_replica_to_existing_trusted_does_not_remove_trusted) {
- EXPECT_EQ(std::string("5:1/2/3/4/u,0/1/2/3/t|"),
- mergeBucketLists("0:5/1/2/3", "0:5/1/2/3|1:5/2/3/4", true));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, batch_update_from_distributor_change_does_not_mark_diverging_replicas_as_trusted) {
- // This differs from batch_update_of_existing_diverging_replicas_does_not_mark_any_as_trusted
- // in that _all_ content nodes are considered outdated when distributor changes take place,
- // and therefore a slightly different code path is taken. In particular, bucket info for
- // outdated nodes gets removed before possibly being re-added (if present in the bucket info
- // response).
- EXPECT_EQ(
- std::string("5:1/7/8/9/u,0/1/2/3/u|"),
- mergeBucketLists(
- lib::ClusterState("distributor:2 storage:3"),
- "0:5/1/2/3|1:5/7/8/9",
- lib::ClusterState("distributor:1 storage:3"),
- "0:5/1/2/3|1:5/7/8/9", true));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-// TODO remove on Vespa 8 - this is a workaround for https://github.com/vespa-engine/vespa/issues/8475
-TEST_F(LegacyBucketDBUpdaterTest, global_distribution_hash_falls_back_to_legacy_format_upon_request_rejection) {
- std::string distConfig(getDistConfig6Nodes2Groups());
- setDistribution(distConfig);
-
- const vespalib::string current_hash = "(0d*|*(0;0;1;2)(1;3;4;5))";
- const vespalib::string legacy_hash = "(0d3|3|*(0;0;1;2)(1;3;4;5))";
-
- setSystemState(lib::ClusterState("distributor:6 storage:6"));
- ASSERT_EQ(messageCount(6), _sender.commands().size());
-
- api::RequestBucketInfoCommand* global_req = nullptr;
- for (auto& cmd : _sender.commands()) {
- auto& req_cmd = dynamic_cast<api::RequestBucketInfoCommand&>(*cmd);
- if (req_cmd.getBucketSpace() == document::FixedBucketSpaces::global_space()) {
- global_req = &req_cmd;
- break;
- }
- }
- ASSERT_TRUE(global_req != nullptr);
- ASSERT_EQ(current_hash, global_req->getDistributionHash());
-
- auto reply = std::make_shared<api::RequestBucketInfoReply>(*global_req);
- reply->setResult(api::ReturnCode::REJECTED);
- getBucketDBUpdater().onRequestBucketInfoReply(reply);
-
- getClock().addSecondsToTime(10);
- getBucketDBUpdater().resendDelayedMessages();
-
- // Should now be a resent request with legacy distribution hash
- ASSERT_EQ(messageCount(6) + 1, _sender.commands().size());
- auto& legacy_req = dynamic_cast<api::RequestBucketInfoCommand&>(*_sender.commands().back());
- ASSERT_EQ(legacy_hash, legacy_req.getDistributionHash());
-
- // Now if we reject it _again_ we should cycle back to the current hash
- // in case it wasn't a hash-based rejection after all. And the circle of life continues.
- reply = std::make_shared<api::RequestBucketInfoReply>(legacy_req);
- reply->setResult(api::ReturnCode::REJECTED);
- getBucketDBUpdater().onRequestBucketInfoReply(reply);
-
- getClock().addSecondsToTime(10);
- getBucketDBUpdater().resendDelayedMessages();
-
- ASSERT_EQ(messageCount(6) + 2, _sender.commands().size());
- auto& new_current_req = dynamic_cast<api::RequestBucketInfoCommand&>(*_sender.commands().back());
- ASSERT_EQ(current_hash, new_current_req.getDistributionHash());
-}
-
-namespace {
-
-template <typename Func>
-void for_each_bucket(const BucketDatabase& db, const document::BucketSpace& space, Func&& f) {
- BucketId last(0);
- auto e = db.getNext(last);
- while (e.valid()) {
- f(space, e);
- e = db.getNext(e.getBucketId());
- }
-}
-
-template <typename Func>
-void for_each_bucket(const DistributorBucketSpaceRepo& repo, Func&& f) {
- for (const auto& space : repo) {
- for_each_bucket(space.second->getBucketDatabase(), space.first, f);
- }
-}
-
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, non_owned_buckets_moved_to_read_only_db_on_ownership_change) {
- getBucketDBUpdater().set_stale_reads_enabled(true);
-
- lib::ClusterState initial_state("distributor:1 storage:4"); // All buckets owned by us by definition
- set_cluster_state_bundle(lib::ClusterStateBundle(initial_state, {}, false)); // Skip activation step for simplicity
-
- ASSERT_EQ(messageCount(4), _sender.commands().size());
- constexpr uint32_t n_buckets = 10;
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(initial_state, messageCount(4), n_buckets));
- _sender.clear();
-
- EXPECT_EQ(size_t(n_buckets), mutable_default_db().size());
- EXPECT_EQ(size_t(n_buckets), mutable_global_db().size());
- EXPECT_EQ(size_t(0), read_only_default_db().size());
- EXPECT_EQ(size_t(0), read_only_global_db().size());
-
- lib::ClusterState pending_state("distributor:2 storage:4");
-
- std::unordered_set<Bucket, Bucket::hash> buckets_not_owned_in_pending_state;
- for_each_bucket(mutable_repo(), [&](const auto& space, const auto& entry) {
- if (!getDistributorBucketSpace().owns_bucket_in_state(pending_state, entry.getBucketId())) {
- buckets_not_owned_in_pending_state.insert(Bucket(space, entry.getBucketId()));
- }
- });
- EXPECT_FALSE(buckets_not_owned_in_pending_state.empty());
-
- set_cluster_state_bundle(lib::ClusterStateBundle(pending_state, {}, true)); // Now requires activation
-
- const auto buckets_not_owned_per_space = (buckets_not_owned_in_pending_state.size() / 2); // 2 spaces
- const auto expected_mutable_buckets = n_buckets - buckets_not_owned_per_space;
- EXPECT_EQ(expected_mutable_buckets, mutable_default_db().size());
- EXPECT_EQ(expected_mutable_buckets, mutable_global_db().size());
- EXPECT_EQ(buckets_not_owned_per_space, read_only_default_db().size());
- EXPECT_EQ(buckets_not_owned_per_space, read_only_global_db().size());
-
- for_each_bucket(read_only_repo(), [&](const auto& space, const auto& entry) {
- EXPECT_TRUE(buckets_not_owned_in_pending_state.find(Bucket(space, entry.getBucketId()))
- != buckets_not_owned_in_pending_state.end());
- });
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, buckets_no_longer_available_are_not_moved_to_read_only_database) {
- constexpr uint32_t n_buckets = 10;
- // No ownership change, just node down. Test redundancy is 2, so removing 2 nodes will
- // cause some buckets to be entirely unavailable.
- trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4,
- "version:2 distributor:1 storage:4 .0.s:d .1.s:m", n_buckets, 0);
-
- EXPECT_EQ(size_t(0), read_only_default_db().size());
- EXPECT_EQ(size_t(0), read_only_global_db().size());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, non_owned_buckets_purged_when_read_only_support_is_config_disabled) {
- getBucketDBUpdater().set_stale_reads_enabled(false);
-
- lib::ClusterState initial_state("distributor:1 storage:4"); // All buckets owned by us by definition
- set_cluster_state_bundle(lib::ClusterStateBundle(initial_state, {}, false)); // Skip activation step for simplicity
-
- ASSERT_EQ(messageCount(4), _sender.commands().size());
- constexpr uint32_t n_buckets = 10;
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(initial_state, messageCount(4), n_buckets));
- _sender.clear();
-
- // Nothing in read-only DB after first bulk load of buckets.
- EXPECT_EQ(size_t(0), read_only_default_db().size());
- EXPECT_EQ(size_t(0), read_only_global_db().size());
-
- lib::ClusterState pending_state("distributor:2 storage:4");
- setSystemState(pending_state);
- // No buckets should be moved into read only db after ownership changes.
- EXPECT_EQ(size_t(0), read_only_default_db().size());
- EXPECT_EQ(size_t(0), read_only_global_db().size());
-}
-
-void LegacyBucketDBUpdaterTest::trigger_completed_but_not_yet_activated_transition(
- vespalib::stringref initial_state_str,
- uint32_t initial_buckets,
- uint32_t initial_expected_msgs,
- vespalib::stringref pending_state_str,
- uint32_t pending_buckets,
- uint32_t pending_expected_msgs)
-{
- lib::ClusterState initial_state(initial_state_str);
- setSystemState(initial_state);
- ASSERT_EQ(messageCount(initial_expected_msgs), _sender.commands().size());
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(
- initial_state, messageCount(initial_expected_msgs), initial_buckets));
- _sender.clear();
-
- lib::ClusterState pending_state(pending_state_str); // Ownership change
- set_cluster_state_bundle(lib::ClusterStateBundle(pending_state, {}, true));
- ASSERT_EQ(messageCount(pending_expected_msgs), _sender.commands().size());
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(
- pending_state, messageCount(pending_expected_msgs), pending_buckets));
- _sender.clear();
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, deferred_activated_state_does_not_enable_state_until_activation_received) {
- getBucketDBUpdater().set_stale_reads_enabled(true);
- constexpr uint32_t n_buckets = 10;
- ASSERT_NO_FATAL_FAILURE(
- trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4,
- "version:2 distributor:1 storage:4", n_buckets, 4));
-
- // Version should not be switched over yet
- EXPECT_EQ(uint32_t(1), current_distributor_cluster_state_bundle().getVersion());
-
- EXPECT_EQ(uint64_t(0), mutable_default_db().size());
- EXPECT_EQ(uint64_t(0), mutable_global_db().size());
-
- EXPECT_FALSE(activate_cluster_state_version(2));
-
- EXPECT_EQ(uint32_t(2), current_distributor_cluster_state_bundle().getVersion());
- EXPECT_EQ(uint64_t(n_buckets), mutable_default_db().size());
- EXPECT_EQ(uint64_t(n_buckets), mutable_global_db().size());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, read_only_db_cleared_once_pending_state_is_activated) {
- getBucketDBUpdater().set_stale_reads_enabled(true);
- constexpr uint32_t n_buckets = 10;
- ASSERT_NO_FATAL_FAILURE(
- trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4,
- "version:2 distributor:2 storage:4", n_buckets, 0));
- EXPECT_FALSE(activate_cluster_state_version(2));
-
- EXPECT_EQ(uint64_t(0), read_only_default_db().size());
- EXPECT_EQ(uint64_t(0), read_only_global_db().size());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, read_only_db_is_populated_even_when_self_is_marked_down) {
- getBucketDBUpdater().set_stale_reads_enabled(true);
- constexpr uint32_t n_buckets = 10;
- ASSERT_NO_FATAL_FAILURE(
- trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4,
- "version:2 distributor:1 .0.s:d storage:4", n_buckets, 0));
-
- // State not yet activated, so read-only DBs have got all the buckets we used to have.
- EXPECT_EQ(uint64_t(0), mutable_default_db().size());
- EXPECT_EQ(uint64_t(0), mutable_global_db().size());
- EXPECT_EQ(uint64_t(n_buckets), read_only_default_db().size());
- EXPECT_EQ(uint64_t(n_buckets), read_only_global_db().size());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, activate_cluster_state_request_with_mismatching_version_returns_actual_version) {
- getBucketDBUpdater().set_stale_reads_enabled(true);
- constexpr uint32_t n_buckets = 10;
- ASSERT_NO_FATAL_FAILURE(
- trigger_completed_but_not_yet_activated_transition("version:4 distributor:1 storage:4", n_buckets, 4,
- "version:5 distributor:2 storage:4", n_buckets, 0));
-
- EXPECT_TRUE(activate_cluster_state_version(4)); // Too old version
- ASSERT_NO_FATAL_FAILURE(assert_has_activate_cluster_state_reply_with_actual_version(5));
-
- EXPECT_TRUE(activate_cluster_state_version(6)); // More recent version than what has been observed
- ASSERT_NO_FATAL_FAILURE(assert_has_activate_cluster_state_reply_with_actual_version(5));
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, activate_cluster_state_request_without_pending_transition_passes_message_through) {
- getBucketDBUpdater().set_stale_reads_enabled(true);
- constexpr uint32_t n_buckets = 10;
- ASSERT_NO_FATAL_FAILURE(
- trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4,
- "version:2 distributor:1 storage:4", n_buckets, 4));
- // Activate version 2; no pending cluster state after this.
- EXPECT_FALSE(activate_cluster_state_version(2));
-
- // No pending cluster state for version 3, just passed through to be implicitly bounced by state manager.
- // Note: state manager is not modelled in this test, so we just check that the message handler returns
- // false (meaning "didn't take message ownership") and there's no auto-generated reply.
- EXPECT_FALSE(activate_cluster_state_version(3));
- EXPECT_EQ(size_t(0), _sender.replies().size());
-}
-
-// TODO STRIPE disabled benchmark tests are NOT migrated to new test suite
-TEST_F(LegacyBucketDBUpdaterTest, DISABLED_benchmark_bulk_loading_into_empty_db) {
- // Need to trigger an initial edge to complete first bucket scan
- ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(lib::ClusterState("distributor:2 storage:1"),
- messageCount(1), 0));
- _sender.clear();
-
- lib::ClusterState state("distributor:1 storage:1");
- setSystemState(state);
-
- constexpr uint32_t superbuckets = 1u << 16u;
- constexpr uint32_t sub_buckets = 14;
- constexpr uint32_t n_buckets = superbuckets * sub_buckets;
-
- ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size());
- for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) {
- ASSERT_EQ(_sender.command(bsi)->getType(), MessageType::REQUESTBUCKETINFO);
- const auto& req = dynamic_cast<const RequestBucketInfoCommand&>(*_sender.command(bsi));
-
- auto sreply = std::make_shared<RequestBucketInfoReply>(req);
- sreply->setAddress(storageAddress(0));
- auto& vec = sreply->getBucketInfo();
- if (req.getBucketSpace() == FixedBucketSpaces::default_space()) {
- for (uint32_t sb = 0; sb < superbuckets; ++sb) {
- for (uint64_t i = 0; i < sub_buckets; ++i) {
- document::BucketId bucket(48, (i << 32ULL) | sb);
- vec.push_back(api::RequestBucketInfoReply::Entry(bucket, api::BucketInfo(10, 1, 1)));
- }
- }
- }
-
- vespalib::BenchmarkTimer timer(1.0);
- // Global space has no buckets but will serve as a trigger for merging
- // buckets into the DB. This lets us measure the overhead of just this part.
- if (req.getBucketSpace() == FixedBucketSpaces::global_space()) {
- timer.before();
- }
- getBucketDBUpdater().onRequestBucketInfoReply(sreply);
- if (req.getBucketSpace() == FixedBucketSpaces::global_space()) {
- timer.after();
- fprintf(stderr, "Took %g seconds to merge %u buckets into DB\n", timer.min_time(), n_buckets);
- }
- }
-
- EXPECT_EQ(size_t(n_buckets), mutable_default_db().size());
- EXPECT_EQ(size_t(0), mutable_global_db().size());
-}
-
-uint32_t LegacyBucketDBUpdaterTest::populate_bucket_db_via_request_bucket_info_for_benchmarking() {
- // Need to trigger an initial edge to complete first bucket scan
- setAndEnableClusterState(lib::ClusterState("distributor:2 storage:1"), messageCount(1), 0);
- _sender.clear();
-
- lib::ClusterState state("distributor:1 storage:1");
- setSystemState(state);
-
- constexpr uint32_t superbuckets = 1u << 16u;
- constexpr uint32_t sub_buckets = 14;
- constexpr uint32_t n_buckets = superbuckets * sub_buckets;
-
- assert(_bucketSpaces.size() == _sender.commands().size());
- for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) {
- assert(_sender.command(bsi)->getType() == MessageType::REQUESTBUCKETINFO);
- const auto& req = dynamic_cast<const RequestBucketInfoCommand&>(*_sender.command(bsi));
-
- auto sreply = std::make_shared<RequestBucketInfoReply>(req);
- sreply->setAddress(storageAddress(0));
- auto& vec = sreply->getBucketInfo();
- if (req.getBucketSpace() == FixedBucketSpaces::default_space()) {
- for (uint32_t sb = 0; sb < superbuckets; ++sb) {
- for (uint64_t i = 0; i < sub_buckets; ++i) {
- document::BucketId bucket(48, (i << 32ULL) | sb);
- vec.push_back(api::RequestBucketInfoReply::Entry(bucket, api::BucketInfo(10, 1, 1)));
- }
- }
- }
- getBucketDBUpdater().onRequestBucketInfoReply(sreply);
- }
-
- assert(mutable_default_db().size() == n_buckets);
- assert(mutable_global_db().size() == 0);
- return n_buckets;
-}
-
-TEST_F(LegacyBucketDBUpdaterTest, DISABLED_benchmark_removing_buckets_for_unavailable_storage_nodes) {
- const uint32_t n_buckets = populate_bucket_db_via_request_bucket_info_for_benchmarking();
-
- lib::ClusterState no_op_state("distributor:1 storage:1 .0.s:m"); // Removing all buckets via ownership
- vespalib::BenchmarkTimer timer(1.0);
- timer.before();
- setSystemState(no_op_state);
- timer.after();
- fprintf(stderr, "Took %g seconds to scan and remove %u buckets\n", timer.min_time(), n_buckets);
-}
-
-TEST_F(LegacyBucketDBUpdaterTest, DISABLED_benchmark_no_buckets_removed_during_node_remover_db_pass) {
- const uint32_t n_buckets = populate_bucket_db_via_request_bucket_info_for_benchmarking();
-
- // TODO this benchmark is void if we further restrict the pruning elision logic to allow
- // elision when storage nodes come online.
- lib::ClusterState no_op_state("distributor:1 storage:2"); // Not removing any buckets
- vespalib::BenchmarkTimer timer(1.0);
- timer.before();
- setSystemState(no_op_state);
- timer.after();
- fprintf(stderr, "Took %g seconds to scan %u buckets with no-op action\n", timer.min_time(), n_buckets);
-}
-
-TEST_F(LegacyBucketDBUpdaterTest, DISABLED_benchmark_all_buckets_removed_during_node_remover_db_pass) {
- const uint32_t n_buckets = populate_bucket_db_via_request_bucket_info_for_benchmarking();
-
- lib::ClusterState no_op_state("distributor:1 storage:1 .0.s:m"); // Removing all buckets via all replicas gone
- vespalib::BenchmarkTimer timer(1.0);
- timer.before();
- setSystemState(no_op_state);
- timer.after();
- fprintf(stderr, "Took %g seconds to scan and remove %u buckets\n", timer.min_time(), n_buckets);
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_getter_is_non_null_only_when_state_is_pending) {
- auto initial_baseline = std::make_shared<lib::ClusterState>("distributor:1 storage:2 .0.s:d");
- auto initial_default = std::make_shared<lib::ClusterState>("distributor:1 storage:2 .0.s:m");
-
- lib::ClusterStateBundle initial_bundle(*initial_baseline, {{FixedBucketSpaces::default_space(), initial_default},
- {FixedBucketSpaces::global_space(), initial_baseline}});
- set_cluster_state_bundle(initial_bundle);
-
- auto* state = getBucketDBUpdater().pendingClusterStateOrNull(FixedBucketSpaces::default_space());
- ASSERT_TRUE(state != nullptr);
- EXPECT_EQ(*initial_default, *state);
-
- state = getBucketDBUpdater().pendingClusterStateOrNull(FixedBucketSpaces::global_space());
- ASSERT_TRUE(state != nullptr);
- EXPECT_EQ(*initial_baseline, *state);
-
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(*initial_baseline, messageCount(1), 0));
-
- state = getBucketDBUpdater().pendingClusterStateOrNull(FixedBucketSpaces::default_space());
- EXPECT_TRUE(state == nullptr);
-
- state = getBucketDBUpdater().pendingClusterStateOrNull(FixedBucketSpaces::global_space());
- EXPECT_TRUE(state == nullptr);
-}
-
-struct LegacyBucketDBUpdaterSnapshotTest : LegacyBucketDBUpdaterTest {
- lib::ClusterState empty_state;
- std::shared_ptr<lib::ClusterState> initial_baseline;
- std::shared_ptr<lib::ClusterState> initial_default;
- lib::ClusterStateBundle initial_bundle;
- Bucket default_bucket;
- Bucket global_bucket;
-
- LegacyBucketDBUpdaterSnapshotTest()
- : LegacyBucketDBUpdaterTest(),
- empty_state(),
- initial_baseline(std::make_shared<lib::ClusterState>("distributor:1 storage:2 .0.s:d")),
- initial_default(std::make_shared<lib::ClusterState>("distributor:1 storage:2 .0.s:m")),
- initial_bundle(*initial_baseline, {{FixedBucketSpaces::default_space(), initial_default},
- {FixedBucketSpaces::global_space(), initial_baseline}}),
- default_bucket(FixedBucketSpaces::default_space(), BucketId(16, 1234)),
- global_bucket(FixedBucketSpaces::global_space(), BucketId(16, 1234))
- {
- }
- ~LegacyBucketDBUpdaterSnapshotTest() override;
-
- void SetUp() override {
- LegacyBucketDBUpdaterTest::SetUp();
- getBucketDBUpdater().set_stale_reads_enabled(true);
- };
-
- // Assumes that the distributor owns all buckets, so it may choose any arbitrary bucket in the bucket space
- uint32_t buckets_in_snapshot_matching_current_db(DistributorBucketSpaceRepo& repo, BucketSpace bucket_space) {
- auto rs = getBucketDBUpdater().read_snapshot_for_bucket(Bucket(bucket_space, BucketId(16, 1234)));
- if (!rs.is_routable()) {
- return 0;
- }
- auto guard = rs.steal_read_guard();
- uint32_t found_buckets = 0;
- for_each_bucket(repo, [&](const auto& space, const auto& entry) {
- if (space == bucket_space) {
- auto entries = guard->find_parents_and_self(entry.getBucketId());
- if (entries.size() == 1) {
- ++found_buckets;
- }
- }
- });
- return found_buckets;
- }
-};
-
-LegacyBucketDBUpdaterSnapshotTest::~LegacyBucketDBUpdaterSnapshotTest() = default;
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterSnapshotTest, default_space_snapshot_prior_to_activated_state_is_non_routable) {
- auto rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket);
- EXPECT_FALSE(rs.is_routable());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterSnapshotTest, global_space_snapshot_prior_to_activated_state_is_non_routable) {
- auto rs = getBucketDBUpdater().read_snapshot_for_bucket(global_bucket);
- EXPECT_FALSE(rs.is_routable());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterSnapshotTest, read_snapshot_returns_appropriate_cluster_states) {
- set_cluster_state_bundle(initial_bundle);
- // State currently pending, empty initial state is active
-
- auto def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket);
- EXPECT_EQ(def_rs.context().active_cluster_state()->toString(), empty_state.toString());
- EXPECT_EQ(def_rs.context().default_active_cluster_state()->toString(), empty_state.toString());
- ASSERT_TRUE(def_rs.context().has_pending_state_transition());
- EXPECT_EQ(def_rs.context().pending_cluster_state()->toString(), initial_default->toString());
-
- auto global_rs = getBucketDBUpdater().read_snapshot_for_bucket(global_bucket);
- EXPECT_EQ(global_rs.context().active_cluster_state()->toString(), empty_state.toString());
- EXPECT_EQ(global_rs.context().default_active_cluster_state()->toString(), empty_state.toString());
- ASSERT_TRUE(global_rs.context().has_pending_state_transition());
- EXPECT_EQ(global_rs.context().pending_cluster_state()->toString(), initial_baseline->toString());
-
- ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(*initial_baseline, messageCount(1), 0));
- // State now activated, no pending
-
- def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket);
- EXPECT_EQ(def_rs.context().active_cluster_state()->toString(), initial_default->toString());
- EXPECT_EQ(def_rs.context().default_active_cluster_state()->toString(), initial_default->toString());
- EXPECT_FALSE(def_rs.context().has_pending_state_transition());
-
- global_rs = getBucketDBUpdater().read_snapshot_for_bucket(global_bucket);
- EXPECT_EQ(global_rs.context().active_cluster_state()->toString(), initial_baseline->toString());
- EXPECT_EQ(global_rs.context().default_active_cluster_state()->toString(), initial_default->toString());
- EXPECT_FALSE(global_rs.context().has_pending_state_transition());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterSnapshotTest, snapshot_with_no_pending_state_transition_returns_mutable_db_guard) {
- constexpr uint32_t n_buckets = 10;
- ASSERT_NO_FATAL_FAILURE(
- trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4,
- "version:2 distributor:1 storage:4", n_buckets, 4));
- EXPECT_FALSE(activate_cluster_state_version(2));
- EXPECT_EQ(buckets_in_snapshot_matching_current_db(mutable_repo(), FixedBucketSpaces::default_space()),
- n_buckets);
- EXPECT_EQ(buckets_in_snapshot_matching_current_db(mutable_repo(), FixedBucketSpaces::global_space()),
- n_buckets);
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterSnapshotTest, snapshot_returns_unroutable_for_non_owned_bucket_in_current_state) {
- ASSERT_NO_FATAL_FAILURE(
- trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4,
- "version:2 distributor:2 .0.s:d storage:4", 0, 0));
- EXPECT_FALSE(activate_cluster_state_version(2));
- // We're down in state 2 and therefore do not own any buckets
- auto def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket);
- EXPECT_FALSE(def_rs.is_routable());
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterSnapshotTest, snapshot_with_pending_state_returns_read_only_guard_for_bucket_only_owned_in_current_state) {
- constexpr uint32_t n_buckets = 10;
- ASSERT_NO_FATAL_FAILURE(
- trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4,
- "version:2 distributor:2 .0.s:d storage:4", 0, 0));
- EXPECT_EQ(buckets_in_snapshot_matching_current_db(read_only_repo(), FixedBucketSpaces::default_space()),
- n_buckets);
- EXPECT_EQ(buckets_in_snapshot_matching_current_db(read_only_repo(), FixedBucketSpaces::global_space()),
- n_buckets);
-}
-
-// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest
-TEST_F(LegacyBucketDBUpdaterSnapshotTest, snapshot_is_unroutable_if_stale_reads_disabled_and_bucket_not_owned_in_pending_state) {
- getBucketDBUpdater().set_stale_reads_enabled(false);
- constexpr uint32_t n_buckets = 10;
- ASSERT_NO_FATAL_FAILURE(
- trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4,
- "version:2 distributor:2 .0.s:d storage:4", 0, 0));
- auto def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket);
- EXPECT_FALSE(def_rs.is_routable());
-}
-
-}
diff --git a/storage/src/tests/distributor/legacy_distributor_test.cpp b/storage/src/tests/distributor/legacy_distributor_test.cpp
deleted file mode 100644
index 90d64ddb130..00000000000
--- a/storage/src/tests/distributor/legacy_distributor_test.cpp
+++ /dev/null
@@ -1,1326 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/storage/distributor/idealstatemetricsset.h>
-#include <vespa/storageapi/message/persistence.h>
-#include <vespa/storageapi/message/bucketsplitting.h>
-#include <vespa/storageapi/message/visitor.h>
-#include <vespa/storageapi/message/removelocation.h>
-#include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h>
-#include <tests/distributor/distributortestutil.h>
-#include <vespa/document/bucket/fixed_bucket_spaces.h>
-#include <vespa/document/fieldset/fieldsets.h>
-#include <vespa/document/test/make_document_bucket.h>
-#include <vespa/document/test/make_bucket_space.h>
-#include <vespa/storage/config/config-stor-distributormanager.h>
-#include <vespa/storage/distributor/top_level_distributor.h>
-#include <vespa/storage/distributor/distributor_stripe.h>
-#include <vespa/storage/distributor/distributor_status.h>
-#include <vespa/storage/distributor/distributor_bucket_space.h>
-#include <vespa/storage/distributor/distributormetricsset.h>
-#include <vespa/vespalib/text/stringtokenizer.h>
-#include <vespa/metrics/updatehook.h>
-#include <thread>
-#include <vespa/vespalib/gtest/gtest.h>
-#include <gmock/gmock.h>
-
-using document::test::makeDocumentBucket;
-using document::test::makeBucketSpace;
-using document::FixedBucketSpaces;
-using document::BucketSpace;
-using document::Bucket;
-using document::BucketId;
-using namespace ::testing;
-
-namespace storage::distributor {
-
-// TODO STRIPE: Remove this test when legacy mode is gone.
-struct LegacyDistributorTest : Test, DistributorTestUtil {
- LegacyDistributorTest();
- ~LegacyDistributorTest() override;
-
- // TODO handle edge case for window between getnodestate reply already
- // sent and new request not yet received
-
- void assertBucketSpaceStats(size_t expBucketPending, size_t expBucketTotal, uint16_t node, const vespalib::string &bucketSpace,
- const BucketSpacesStatsProvider::PerNodeBucketSpacesStats &stats);
- std::vector<document::BucketSpace> _bucketSpaces;
-
- void SetUp() override {
- createLinks();
- _bucketSpaces = getBucketSpaces();
- };
-
- void TearDown() override {
- close();
- }
-
- // Simple type aliases to make interfacing with certain utility functions
- // easier. Note that this is only for readability and does not provide any
- // added type safety.
- using NodeCount = int;
- using Redundancy = int;
-
- using ConfigBuilder = vespa::config::content::core::StorDistributormanagerConfigBuilder;
-
- void configureDistributor(const ConfigBuilder& config) {
- getConfig().configure(config);
- _distributor->enable_next_config_if_changed();
- }
-
- auto currentReplicaCountingMode() const noexcept {
- return _distributor->bucket_db_metric_updater().getMinimumReplicaCountingMode();
- }
-
- std::string testOp(std::shared_ptr<api::StorageMessage> msg)
- {
- _distributor->handleMessage(msg);
-
- std::string tmp = _sender.getCommands();
- _sender.clear();
- return tmp;
- }
-
- void tickDistributorNTimes(uint32_t n) {
- for (uint32_t i = 0; i < n; ++i) {
- tick();
- }
- }
-
- typedef bool ResetTrusted;
-
- std::string updateBucketDB(const std::string& firstState,
- const std::string& secondState,
- bool resetTrusted = false)
- {
- std::vector<std::string> states(toVector<std::string>(firstState, secondState));
-
- for (uint32_t i = 0; i < states.size(); ++i) {
- std::vector<uint16_t> removedNodes;
- std::vector<BucketCopy> changedNodes;
-
- vespalib::StringTokenizer tokenizer(states[i], ",");
- for (uint32_t j = 0; j < tokenizer.size(); ++j) {
- vespalib::StringTokenizer tokenizer2(tokenizer[j], ":");
-
- bool trusted = false;
- if (tokenizer2.size() > 2) {
- trusted = true;
- }
-
- uint16_t node = atoi(tokenizer2[0].data());
- if (tokenizer2[1] == "r") {
- removedNodes.push_back(node);
- } else {
- uint32_t checksum = atoi(tokenizer2[1].data());
- changedNodes.push_back(
- BucketCopy(
- i + 1,
- node,
- api::BucketInfo(
- checksum,
- checksum / 2,
- checksum / 4)).setTrusted(trusted));
- }
- }
-
- operation_context().remove_nodes_from_bucket_database(makeDocumentBucket(document::BucketId(16, 1)), removedNodes);
-
- uint32_t flags(DatabaseUpdate::CREATE_IF_NONEXISTING
- | (resetTrusted ? DatabaseUpdate::RESET_TRUSTED : 0));
-
- operation_context().update_bucket_database(makeDocumentBucket(document::BucketId(16, 1)),
- changedNodes,
- flags);
- }
-
- std::string retVal = dumpBucket(document::BucketId(16, 1));
- getBucketDatabase().clear();
- return retVal;
- }
-
- size_t explicit_node_state_reply_send_invocations() const noexcept {
- return _node->getNodeStateUpdater().explicit_node_state_reply_send_invocations();
- }
-
- StatusReporterDelegate& distributor_status_delegate() {
- // TODO STRIPE
- return _distributor->_stripe->_distributorStatusDelegate;
- }
-
- framework::TickingThreadPool& distributor_thread_pool() {
- return _distributor->_threadPool;
- }
-
- const std::vector<std::shared_ptr<DistributorStatus>>& distributor_status_todos() {
- // TODO STRIPE
- return _distributor->_stripe->_statusToDo;
- }
-
- TopLevelDistributor::MetricUpdateHook distributor_metric_update_hook() {
- return _distributor->_metricUpdateHook;
- }
-
- SimpleMaintenanceScanner::PendingMaintenanceStats distributor_maintenance_stats() {
- return _distributor->pending_maintenance_stats();
- }
-
- BucketSpacesStatsProvider::PerNodeBucketSpacesStats distributor_bucket_spaces_stats() {
- return _distributor->getBucketSpacesStats();
- }
-
- DistributorHostInfoReporter& distributor_host_info_reporter() {
- return _distributor->_hostInfoReporter;
- }
-
- bool distributor_handle_message(const std::shared_ptr<api::StorageMessage>& msg) {
- return _distributor->handleMessage(msg);
- }
-
- uint64_t db_sample_interval_sec() const noexcept {
- return std::chrono::duration_cast<std::chrono::seconds>(_distributor->db_memory_sample_interval()).count();
- }
-
- void configure_stale_reads_enabled(bool enabled) {
- ConfigBuilder builder;
- builder.allowStaleReadsDuringClusterStateTransitions = enabled;
- configureDistributor(builder);
- }
-
- void configure_update_fast_path_restart_enabled(bool enabled) {
- ConfigBuilder builder;
- builder.restartWithFastUpdatePathIfAllGetTimestampsAreConsistent = enabled;
- configureDistributor(builder);
- }
-
- void configure_merge_operations_disabled(bool disabled) {
- ConfigBuilder builder;
- builder.mergeOperationsDisabled = disabled;
- configureDistributor(builder);
- }
-
- void configure_use_weak_internal_read_consistency(bool use_weak) {
- ConfigBuilder builder;
- builder.useWeakInternalReadConsistencyForClientGets = use_weak;
- configureDistributor(builder);
- }
-
- void configure_metadata_update_phase_enabled(bool enabled) {
- ConfigBuilder builder;
- builder.enableMetadataOnlyFetchPhaseForInconsistentUpdates = enabled;
- configureDistributor(builder);
- }
-
- void configure_prioritize_global_bucket_merges(bool enabled) {
- ConfigBuilder builder;
- builder.prioritizeGlobalBucketMerges = enabled;
- configureDistributor(builder);
- }
-
- void configure_max_activation_inhibited_out_of_sync_groups(uint32_t n_groups) {
- ConfigBuilder builder;
- builder.maxActivationInhibitedOutOfSyncGroups = n_groups;
- configureDistributor(builder);
- }
-
- void configureMaxClusterClockSkew(int seconds);
- void replyToSingleRequestBucketInfoCommandWith1Bucket();
- void sendDownDummyRemoveCommand();
- void assertSingleBouncedRemoveReplyPresent();
- void assertNoMessageBounced();
- void configure_mutation_sequencing(bool enabled);
- void configure_merge_busy_inhibit_duration(int seconds);
- void do_test_pending_merge_getnodestate_reply_edge(BucketSpace space);
-
- void set_up_and_start_get_op_with_stale_reads_enabled(bool enabled);
-};
-
-LegacyDistributorTest::LegacyDistributorTest()
- : Test(),
- DistributorTestUtil(),
- _bucketSpaces()
-{
-}
-
-LegacyDistributorTest::~LegacyDistributorTest() = default;
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, operation_generation) {
- setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1");
-
- document::BucketId bid;
- addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t");
-
- EXPECT_EQ("Remove", testOp(std::make_shared<api::RemoveCommand>(
- makeDocumentBucket(bid),
- document::DocumentId("id:m:test:n=1:foo"),
- api::Timestamp(1234))));
-
- auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "foo", "bar", "");
- cmd->addBucketToBeVisited(document::BucketId(16, 1));
- cmd->addBucketToBeVisited(document::BucketId());
-
- EXPECT_EQ("Visitor Create", testOp(cmd));
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, operations_generated_and_started_without_duplicates) {
- setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1");
-
- for (uint32_t i = 0; i < 6; ++i) {
- addNodesToBucketDB(document::BucketId(16, i), "0=1");
- }
-
- tickDistributorNTimes(20);
-
- ASSERT_FALSE(tick());
-
- ASSERT_EQ(6, _sender.commands().size());
-}
-
-
-// Migrated to DistributorStripeTest
-// Migrated to TopLevelDistributorTest
-TEST_F(LegacyDistributorTest, recovery_mode_on_cluster_state_change) {
- setupDistributor(Redundancy(1), NodeCount(2),
- "storage:1 .0.s:d distributor:1");
- enableDistributorClusterState("storage:1 distributor:1");
-
- EXPECT_TRUE(distributor_is_in_recovery_mode());
- for (uint32_t i = 0; i < 3; ++i) {
- addNodesToBucketDB(document::BucketId(16, i), "0=1");
- }
- for (int i = 0; i < 3; ++i) {
- tick();
- EXPECT_TRUE(distributor_is_in_recovery_mode());
- }
- tick();
- EXPECT_FALSE(distributor_is_in_recovery_mode());
-
- enableDistributorClusterState("storage:2 distributor:1");
- EXPECT_TRUE(distributor_is_in_recovery_mode());
-}
-
-TEST_F(LegacyDistributorTest, distributor_considered_initialized_once_self_observed_up) {
- setupDistributor(Redundancy(1), NodeCount(2), "distributor:1 .0.s:d storage:1"); // We're down D:
- EXPECT_FALSE(_distributor->done_initializing());
- enableDistributorClusterState("distributor:1 storage:1"); // We're up :D
- EXPECT_TRUE(_distributor->done_initializing());
- enableDistributorClusterState("distributor:1 .0.s:d storage:1"); // And down again :I but that does not change init state
- EXPECT_TRUE(_distributor->done_initializing());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, operations_are_throttled) {
- setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1");
- getConfig().setMinPendingMaintenanceOps(1);
- getConfig().setMaxPendingMaintenanceOps(1);
-
- for (uint32_t i = 0; i < 6; ++i) {
- addNodesToBucketDB(document::BucketId(16, i), "0=1");
- }
- tickDistributorNTimes(20);
- ASSERT_EQ(1, _sender.commands().size());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, handle_unknown_maintenance_reply) {
- setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1");
-
- {
- auto cmd = std::make_shared<api::SplitBucketCommand>(makeDocumentBucket(document::BucketId(16, 1234)));
- auto reply = std::make_shared<api::SplitBucketReply>(*cmd);
- ASSERT_TRUE(_distributor->handleReply(reply));
- }
-
- {
- // RemoveLocationReply must be treated as a maintenance reply since
- // it's what GC is currently built around.
- auto cmd = std::make_shared<api::RemoveLocationCommand>(
- "false", makeDocumentBucket(document::BucketId(30, 1234)));
- auto reply = std::shared_ptr<api::StorageReply>(cmd->makeReply());
- ASSERT_TRUE(_distributor->handleReply(reply));
- }
-}
-
-// Migrated to TopLevelDistributorTest
-TEST_F(LegacyDistributorTest, contains_time_statement) {
- setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1");
-
- EXPECT_FALSE(getConfig().containsTimeStatement(""));
- EXPECT_FALSE(getConfig().containsTimeStatement("testdoctype1"));
- EXPECT_FALSE(getConfig().containsTimeStatement("testdoctype1.headerfield > 42"));
- EXPECT_TRUE(getConfig().containsTimeStatement("testdoctype1.headerfield > now()"));
- EXPECT_TRUE(getConfig().containsTimeStatement("testdoctype1.headerfield > now() - 3600"));
- EXPECT_TRUE(getConfig().containsTimeStatement("testdoctype1.headerfield == now() - 3600"));
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, update_bucket_database) {
- enableDistributorClusterState("distributor:1 storage:3");
-
- EXPECT_EQ("BucketId(0x4000000000000001) : "
- "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false), "
- "node(idx=1,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false)",
- updateBucketDB("0:456,1:456,2:789", "2:r"));
-
- EXPECT_EQ("BucketId(0x4000000000000001) : "
- "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false), "
- "node(idx=2,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false), "
- "node(idx=1,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false)",
- updateBucketDB("0:456,1:456", "2:456"));
-
- EXPECT_EQ("BucketId(0x4000000000000001) : "
- "node(idx=0,crc=0x315,docs=394/394,bytes=197/197,trusted=false,active=false,ready=false), "
- "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=false,active=false,ready=false), "
- "node(idx=1,crc=0x34a,docs=421/421,bytes=210/210,trusted=false,active=false,ready=false)",
- updateBucketDB("0:456:t,1:456:t,2:123", "0:789,1:842,2:333"));
-
- EXPECT_EQ("BucketId(0x4000000000000001) : "
- "node(idx=0,crc=0x315,docs=394/394,bytes=197/197,trusted=true,active=false,ready=false), "
- "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=false,active=false,ready=false), "
- "node(idx=1,crc=0x315,docs=394/394,bytes=197/197,trusted=true,active=false,ready=false)",
- updateBucketDB("0:456:t,1:456:t,2:123", "0:789,1:789,2:333"));
-
- EXPECT_EQ("BucketId(0x4000000000000001) : "
- "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=true,active=false,ready=false)",
- updateBucketDB("0:456:t,1:456:t", "0:r,1:r,2:333"));
-
- // Copies are in sync so should still be trusted even if explicitly reset.
- EXPECT_EQ("BucketId(0x4000000000000001) : "
- "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false), "
- "node(idx=2,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false), "
- "node(idx=1,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false)",
- updateBucketDB("0:456,1:456", "2:456", ResetTrusted(true)));
-
- // When resetting, first inserted copy should not end up as implicitly trusted.
- EXPECT_EQ("BucketId(0x4000000000000001) : "
- "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=false,active=false,ready=false), "
- "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=false,active=false,ready=false)",
- updateBucketDB("0:456", "2:333", ResetTrusted(true)));
-}
-
-namespace {
-
-using namespace framework::defaultimplementation;
-
-class StatusRequestThread : public framework::Runnable {
- StatusReporterDelegate& _reporter;
- std::string _result;
-public:
- explicit StatusRequestThread(StatusReporterDelegate& reporter)
- : _reporter(reporter)
- {}
- void run(framework::ThreadHandle&) override {
- framework::HttpUrlPath path("/distributor?page=buckets");
- std::ostringstream stream;
- _reporter.reportStatus(stream, path);
- _result = stream.str();
- }
-
- std::string getResult() const {
- return _result;
- }
-};
-
-}
-
-// Migrated to TopLevelDistributorTest
-TEST_F(LegacyDistributorTest, tick_processes_status_requests) {
- setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1");
-
- addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t");
-
- // Must go via delegate since reportStatus is now just a rendering
- // function and not a request enqueuer (see Distributor::handleStatusRequest).
- StatusRequestThread thread(distributor_status_delegate());
- FakeClock clock;
- ThreadPoolImpl pool(clock);
- int ticksBeforeWait = 1;
- framework::Thread::UP tp(pool.startThread(thread, "statustest", 5ms, 5s, ticksBeforeWait));
-
- while (true) {
- std::this_thread::sleep_for(1ms);
- framework::TickingLockGuard guard(distributor_thread_pool().freezeCriticalTicks());
- if (!distributor_status_todos().empty()) {
- break;
- }
-
- }
- ASSERT_TRUE(tick());
-
- tp->interruptAndJoin();
-
- EXPECT_THAT(thread.getResult(), HasSubstr("BucketId(0x4000000000000001)"));
-}
-
-// Migrated to TopLevelDistributorTest
-TEST_F(LegacyDistributorTest, metric_update_hook_updates_pending_maintenance_metrics) {
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
- // To ensure we count all operations, not just those fitting within the
- // pending window.
- getConfig().setMinPendingMaintenanceOps(1);
- getConfig().setMaxPendingMaintenanceOps(1);
-
- // 1 bucket must be merged, 1 must be split, 1 should be activated.
- addNodesToBucketDB(document::BucketId(16, 1), "0=2/2/2/t/a,1=1/1/1");
- addNodesToBucketDB(document::BucketId(16, 2),
- "0=100/10000000/200000/t/a,1=100/10000000/200000/t");
- addNodesToBucketDB(document::BucketId(16, 3),
- "0=200/300/400/t,1=200/300/400/t");
-
- // Go many full scanner rounds to check that metrics are set, not
- // added to existing.
- tickDistributorNTimes(50);
-
- // By this point, no hook has been called so the metrics have not been
- // set.
- using MO = MaintenanceOperation;
- {
- const IdealStateMetricSet& metrics(getIdealStateManager().getMetrics());
- EXPECT_EQ(0, metrics.operations[MO::MERGE_BUCKET]->pending.getLast());
- EXPECT_EQ(0, metrics.operations[MO::SPLIT_BUCKET]->pending.getLast());
- EXPECT_EQ(0, metrics.operations[MO::SET_BUCKET_STATE]->pending.getLast());
- EXPECT_EQ(0, metrics.operations[MO::DELETE_BUCKET]->pending.getLast());
- EXPECT_EQ(0, metrics.operations[MO::JOIN_BUCKET]->pending.getLast());
- EXPECT_EQ(0, metrics.operations[MO::GARBAGE_COLLECTION]->pending.getLast());
- }
-
- // Force trigger update hook
- std::mutex l;
- distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l));
- // Metrics should now be updated to the last complete working state
- {
- const IdealStateMetricSet& metrics(getIdealStateManager().getMetrics());
- EXPECT_EQ(1, metrics.operations[MO::MERGE_BUCKET]->pending.getLast());
- EXPECT_EQ(1, metrics.operations[MO::SPLIT_BUCKET]->pending.getLast());
- EXPECT_EQ(1, metrics.operations[MO::SET_BUCKET_STATE]->pending.getLast());
- EXPECT_EQ(0, metrics.operations[MO::DELETE_BUCKET]->pending.getLast());
- EXPECT_EQ(0, metrics.operations[MO::JOIN_BUCKET]->pending.getLast());
- EXPECT_EQ(0, metrics.operations[MO::GARBAGE_COLLECTION]->pending.getLast());
- }
-}
-
-// Migrated to TopLevelDistributorTest
-TEST_F(LegacyDistributorTest, bucket_db_memory_usage_metrics_only_updated_at_fixed_time_intervals) {
- getClock().setAbsoluteTimeInSeconds(1000);
-
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
- addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a,1=2/2/2");
- tickDistributorNTimes(10);
-
- std::mutex l;
- distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l));
- auto* m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes");
- ASSERT_TRUE(m != nullptr);
- auto last_used = m->getLongValue("last");
- EXPECT_GT(last_used, 0);
-
- // Add another bucket to the DB. This should increase the underlying used number of
- // bytes, but this should not be aggregated into the metrics until the sampling time
- // interval has passed. Instead, old metric gauge values should be preserved.
- addNodesToBucketDB(document::BucketId(16, 2), "0=1/1/1/t/a,1=2/2/2");
-
- const auto sample_interval_sec = db_sample_interval_sec();
- getClock().setAbsoluteTimeInSeconds(1000 + sample_interval_sec - 1); // Not there yet.
- tickDistributorNTimes(50);
- distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l));
-
- m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes");
- auto now_used = m->getLongValue("last");
- EXPECT_EQ(now_used, last_used);
-
- getClock().setAbsoluteTimeInSeconds(1000 + sample_interval_sec + 1);
- tickDistributorNTimes(10);
- distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l));
-
- m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes");
- now_used = m->getLongValue("last");
- EXPECT_GT(now_used, last_used);
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, priority_config_is_propagated_to_distributor_configuration) {
- using namespace vespa::config::content::core;
-
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
-
- ConfigBuilder builder;
- builder.priorityMergeMoveToIdealNode = 1;
- builder.priorityMergeOutOfSyncCopies = 2;
- builder.priorityMergeTooFewCopies = 3;
- builder.priorityActivateNoExistingActive = 4;
- builder.priorityActivateWithExistingActive = 5;
- builder.priorityDeleteBucketCopy = 6;
- builder.priorityJoinBuckets = 7;
- builder.prioritySplitDistributionBits = 8;
- builder.prioritySplitLargeBucket = 9;
- builder.prioritySplitInconsistentBucket = 10;
- builder.priorityGarbageCollection = 11;
- builder.priorityMergeGlobalBuckets = 12;
-
- getConfig().configure(builder);
-
- const auto& mp = getConfig().getMaintenancePriorities();
- EXPECT_EQ(1, static_cast<int>(mp.mergeMoveToIdealNode));
- EXPECT_EQ(2, static_cast<int>(mp.mergeOutOfSyncCopies));
- EXPECT_EQ(3, static_cast<int>(mp.mergeTooFewCopies));
- EXPECT_EQ(4, static_cast<int>(mp.activateNoExistingActive));
- EXPECT_EQ(5, static_cast<int>(mp.activateWithExistingActive));
- EXPECT_EQ(6, static_cast<int>(mp.deleteBucketCopy));
- EXPECT_EQ(7, static_cast<int>(mp.joinBuckets));
- EXPECT_EQ(8, static_cast<int>(mp.splitDistributionBits));
- EXPECT_EQ(9, static_cast<int>(mp.splitLargeBucket));
- EXPECT_EQ(10, static_cast<int>(mp.splitInconsistentBucket));
- EXPECT_EQ(11, static_cast<int>(mp.garbageCollection));
- EXPECT_EQ(12, static_cast<int>(mp.mergeGlobalBuckets));
-}
-
-// Migrated to DistributorStripeTest
-// Explicit cluster state edge test added in TopLevelDistributorTest::cluster_state_lifecycle_is_propagated_to_stripes
-TEST_F(LegacyDistributorTest, no_db_resurrection_for_bucket_not_owned_in_pending_state) {
- setupDistributor(Redundancy(1), NodeCount(10), "storage:2 distributor:2");
- lib::ClusterState newState("storage:10 distributor:10");
- auto stateCmd = std::make_shared<api::SetSystemStateCommand>(newState);
- // Force newState into being the pending state. According to the initial
- // state we own the bucket, but according to the pending state, we do
- // not. This must be handled correctly by the database update code.
- getBucketDBUpdater().onSetSystemState(stateCmd);
-
- document::BucketId nonOwnedBucket(16, 3);
- EXPECT_FALSE(getDistributorBucketSpace().get_bucket_ownership_flags(nonOwnedBucket).owned_in_pending_state());
- EXPECT_FALSE(getDistributorBucketSpace().check_ownership_in_pending_and_current_state(nonOwnedBucket).isOwned());
-
- std::vector<BucketCopy> copies;
- copies.emplace_back(1234, 0, api::BucketInfo(0x567, 1, 2));
- operation_context().update_bucket_database(makeDocumentBucket(nonOwnedBucket), copies,
- DatabaseUpdate::CREATE_IF_NONEXISTING);
-
- EXPECT_EQ("NONEXISTING", dumpBucket(nonOwnedBucket));
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, added_db_buckets_without_gc_timestamp_implicitly_get_current_time) {
- setupDistributor(Redundancy(1), NodeCount(10), "storage:2 distributor:2");
- getClock().setAbsoluteTimeInSeconds(101234);
- document::BucketId bucket(16, 7654);
-
- std::vector<BucketCopy> copies;
- copies.emplace_back(1234, 0, api::BucketInfo(0x567, 1, 2));
- operation_context().update_bucket_database(makeDocumentBucket(bucket), copies,
- DatabaseUpdate::CREATE_IF_NONEXISTING);
- BucketDatabase::Entry e(getBucket(bucket));
- EXPECT_EQ(101234, e->getLastGarbageCollectionTime());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, merge_stats_are_accumulated_during_database_iteration) {
- setupDistributor(Redundancy(2), NodeCount(3), "storage:3 distributor:1");
- // Copies out of sync. Not possible for distributor to _reliably_ tell
- // which direction(s) data will flow, so for simplicity assume that we
- // must sync both copies.
- // Note that we mark certain copies as active to prevent the bucketstate
- // checker from pre-empting the merges.
- // -> syncing[0] += 1, syncing[2] += 1
- addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a,2=2/2/2");
- // Must add missing node 2 for bucket
- // -> copyingOut[0] += 1, copyingIn[2] += 1
- addNodesToBucketDB(document::BucketId(16, 2), "0=1/1/1/t/a");
- // Moving from non-ideal node 1 to ideal node 2. Both nodes 0 and 1 will
- // be involved in this merge, but only node 1 will be tagged as source only
- // (i.e. to be deleted after the merge is completed).
- // -> copyingOut[0] += 1, movingOut[1] += 1, copyingIn[2] += 1
- addNodesToBucketDB(document::BucketId(16, 3), "0=2/2/2/t/a,1=2/2/2/t");
-
- // Go many full scanner rounds to check that stats are set, not
- // added to existing.
- tickDistributorNTimes(50);
-
- const auto& stats = distributor_maintenance_stats();
- {
- NodeMaintenanceStats wanted;
- wanted.syncing = 1;
- wanted.copyingOut = 2;
- wanted.total = 3;
- EXPECT_EQ(wanted, stats.perNodeStats.forNode(0, makeBucketSpace()));
- }
- {
- NodeMaintenanceStats wanted;
- wanted.movingOut = 1;
- wanted.total = 1;
- EXPECT_EQ(wanted, stats.perNodeStats.forNode(1, makeBucketSpace()));
- }
- {
- NodeMaintenanceStats wanted;
- wanted.syncing = 1;
- wanted.copyingIn = 2;
- wanted.total = 1;
- EXPECT_EQ(wanted, stats.perNodeStats.forNode(2, makeBucketSpace()));
- }
- auto bucketStats = distributor_bucket_spaces_stats();
- ASSERT_EQ(3, bucketStats.size());
- assertBucketSpaceStats(1, 3, 0, "default", bucketStats);
- assertBucketSpaceStats(0, 1, 1, "default", bucketStats);
- assertBucketSpaceStats(3, 1, 2, "default", bucketStats);
-}
-
-void
-LegacyDistributorTest::assertBucketSpaceStats(size_t expBucketPending, size_t expBucketTotal, uint16_t node,
- const vespalib::string& bucketSpace,
- const BucketSpacesStatsProvider::PerNodeBucketSpacesStats& stats)
-{
- auto nodeItr = stats.find(node);
- ASSERT_TRUE(nodeItr != stats.end());
- ASSERT_EQ(1, nodeItr->second.size());
- auto bucketSpaceItr = nodeItr->second.find(bucketSpace);
- ASSERT_TRUE(bucketSpaceItr != nodeItr->second.end());
- ASSERT_TRUE(bucketSpaceItr->second.valid());
- ASSERT_EQ(expBucketTotal, bucketSpaceItr->second.bucketsTotal());
- ASSERT_EQ(expBucketPending, bucketSpaceItr->second.bucketsPending());
-}
-
-/**
- * Since maintenance operations are prioritized differently, activation
- * pre-empts merging and other ops. If this also implies pre-empting running
- * their state checkers at all, we won't get any statistics from any other
- * operations for the bucket.
- */
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, stats_generated_for_preempted_operations) {
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
- // For this test it suffices to have a single bucket with multiple aspects
- // wrong about it. In this case, let a bucket be both out of sync _and_
- // missing an active copy. This _should_ give a statistic with both nodes 0
- // and 1 requiring a sync. If instead merge stats generation is preempted
- // by activation, we'll see no merge stats at all.
- addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1,1=2/2/2");
- tickDistributorNTimes(50);
- const auto& stats = distributor_maintenance_stats();
- {
- NodeMaintenanceStats wanted;
- wanted.syncing = 1;
- wanted.total = 1;
- EXPECT_EQ(wanted, stats.perNodeStats.forNode(0, makeBucketSpace()));
- }
- {
- NodeMaintenanceStats wanted;
- wanted.syncing = 1;
- wanted.total = 1;
- EXPECT_EQ(wanted, stats.perNodeStats.forNode(1, makeBucketSpace()));
- }
-}
-
-// Migrated to TopLevelDistributorTest
-TEST_F(LegacyDistributorTest, host_info_reporter_config_is_propagated_to_reporter) {
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
-
- // Default is enabled=true.
- EXPECT_TRUE(distributor_host_info_reporter().isReportingEnabled());
-
- ConfigBuilder builder;
- builder.enableHostInfoReporting = false;
- configureDistributor(builder);
-
- EXPECT_FALSE(distributor_host_info_reporter().isReportingEnabled());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, replica_counting_mode_is_configured_to_trusted_by_default) {
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
- EXPECT_EQ(ConfigBuilder::MinimumReplicaCountingMode::TRUSTED, currentReplicaCountingMode());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, replica_counting_mode_config_is_propagated_to_metric_updater) {
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
- ConfigBuilder builder;
- builder.minimumReplicaCountingMode = ConfigBuilder::MinimumReplicaCountingMode::ANY;
- configureDistributor(builder);
- EXPECT_EQ(ConfigBuilder::MinimumReplicaCountingMode::ANY, currentReplicaCountingMode());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, max_consecutively_inhibited_maintenance_ticks_config_is_propagated_to_internal_config) {
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
- ConfigBuilder builder;
- builder.maxConsecutivelyInhibitedMaintenanceTicks = 123;
- getConfig().configure(builder);
- EXPECT_EQ(getConfig().max_consecutively_inhibited_maintenance_ticks(), 123);
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, bucket_activation_is_enabled_by_default) {
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
- EXPECT_FALSE(getConfig().isBucketActivationDisabled());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, bucket_activation_config_is_propagated_to_distributor_configuration) {
- using namespace vespa::config::content::core;
-
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
-
- ConfigBuilder builder;
- builder.disableBucketActivation = true;
- getConfig().configure(builder);
-
- EXPECT_TRUE(getConfig().isBucketActivationDisabled());
-}
-
-void
-LegacyDistributorTest::configureMaxClusterClockSkew(int seconds) {
- using namespace vespa::config::content::core;
-
- ConfigBuilder builder;
- builder.maxClusterClockSkewSec = seconds;
- getConfig().configure(builder);
- _distributor->enable_next_config_if_changed();
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, max_clock_skew_config_is_propagated_to_distributor_config) {
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
-
- configureMaxClusterClockSkew(5);
- EXPECT_EQ(getConfig().getMaxClusterClockSkew(), std::chrono::seconds(5));
-}
-
-namespace {
-
-auto makeDummyRemoveCommand() {
- return std::make_shared<api::RemoveCommand>(
- makeDocumentBucket(document::BucketId(0)),
- document::DocumentId("id:foo:testdoctype1:n=1:foo"),
- api::Timestamp(0));
-}
-
-auto make_dummy_get_command_for_bucket_1() {
- return std::make_shared<api::GetCommand>(
- makeDocumentBucket(document::BucketId(0)),
- document::DocumentId("id:foo:testdoctype1:n=1:foo"),
- document::AllFields::NAME);
-}
-
-}
-
-void LegacyDistributorTest::replyToSingleRequestBucketInfoCommandWith1Bucket() {
- ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size());
- for (uint32_t i = 0; i < _sender.commands().size(); ++i) {
- ASSERT_EQ(api::MessageType::REQUESTBUCKETINFO, _sender.command(i)->getType());
- auto& bucketReq(static_cast<api::RequestBucketInfoCommand&>
- (*_sender.command(i)));
- auto bucketReply = bucketReq.makeReply();
- if (bucketReq.getBucketSpace() == FixedBucketSpaces::default_space()) {
- // Make sure we have a bucket to route our remove op to, or we'd get
- // an immediate reply anyway.
- dynamic_cast<api::RequestBucketInfoReply&>(*bucketReply)
- .getBucketInfo().push_back(
- api::RequestBucketInfoReply::Entry(document::BucketId(1, 1),
- api::BucketInfo(20, 10, 12, 50, 60, true, true)));
- }
- _distributor->handleMessage(std::move(bucketReply));
- }
- _sender.commands().clear();
-}
-
-void LegacyDistributorTest::sendDownDummyRemoveCommand() {
- _distributor->handleMessage(makeDummyRemoveCommand());
-}
-
-void LegacyDistributorTest::assertSingleBouncedRemoveReplyPresent() {
- ASSERT_EQ(1, _sender.replies().size()); // Rejected remove
- ASSERT_EQ(api::MessageType::REMOVE_REPLY, _sender.reply(0)->getType());
- auto& reply(static_cast<api::RemoveReply&>(*_sender.reply(0)));
- ASSERT_EQ(api::ReturnCode::STALE_TIMESTAMP, reply.getResult().getResult());
- _sender.replies().clear();
-}
-
-void LegacyDistributorTest::assertNoMessageBounced() {
- ASSERT_EQ(0, _sender.replies().size());
-}
-
-// Migrated to TopLevelDistributorTest
-TEST_F(LegacyDistributorTest, configured_safe_time_point_rejection_works_end_to_end) {
- setupDistributor(Redundancy(2), NodeCount(2),
- "bits:1 storage:1 distributor:2");
- getClock().setAbsoluteTimeInSeconds(1000);
- configureMaxClusterClockSkew(10);
-
- receive_set_system_state_command("bits:1 storage:1 distributor:1");
- ASSERT_NO_FATAL_FAILURE(replyToSingleRequestBucketInfoCommandWith1Bucket());
- // SetSystemStateCommand sent down chain at this point.
- sendDownDummyRemoveCommand();
- ASSERT_NO_FATAL_FAILURE(assertSingleBouncedRemoveReplyPresent());
-
- // Increment time to first whole second of clock + 10 seconds of skew.
- // Should now not get any feed rejections.
- getClock().setAbsoluteTimeInSeconds(1011);
-
- sendDownDummyRemoveCommand();
- ASSERT_NO_FATAL_FAILURE(assertNoMessageBounced());
-}
-
-void LegacyDistributorTest::configure_mutation_sequencing(bool enabled) {
- using namespace vespa::config::content::core;
-
- ConfigBuilder builder;
- builder.sequenceMutatingOperations = enabled;
- getConfig().configure(builder);
- _distributor->enable_next_config_if_changed();
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, sequencing_config_is_propagated_to_distributor_config) {
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
-
- // Should be enabled by default
- EXPECT_TRUE(getConfig().getSequenceMutatingOperations());
-
- // Explicitly disabled.
- configure_mutation_sequencing(false);
- EXPECT_FALSE(getConfig().getSequenceMutatingOperations());
-
- // Explicitly enabled.
- configure_mutation_sequencing(true);
- EXPECT_TRUE(getConfig().getSequenceMutatingOperations());
-}
-
-void
-LegacyDistributorTest::configure_merge_busy_inhibit_duration(int seconds) {
- using namespace vespa::config::content::core;
-
- ConfigBuilder builder;
- builder.inhibitMergeSendingOnBusyNodeDurationSec = seconds;
- getConfig().configure(builder);
- _distributor->enable_next_config_if_changed();
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, merge_busy_inhibit_duration_config_is_propagated_to_distributor_config) {
- setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
-
- configure_merge_busy_inhibit_duration(7);
- EXPECT_EQ(getConfig().getInhibitMergesOnBusyNodeDuration(), std::chrono::seconds(7));
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, merge_busy_inhibit_duration_is_propagated_to_pending_message_tracker) {
- setupDistributor(Redundancy(2), NodeCount(2), "storage:1 distributor:1");
- addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t");
-
- configure_merge_busy_inhibit_duration(100);
- auto cmd = makeDummyRemoveCommand(); // Remove is for bucket 1
- distributor_handle_message(cmd);
-
- // Should send to content node 0
- ASSERT_EQ(1, _sender.commands().size());
- ASSERT_EQ(api::MessageType::REMOVE, _sender.command(0)->getType());
- auto& fwd_cmd = dynamic_cast<api::RemoveCommand&>(*_sender.command(0));
- auto reply = fwd_cmd.makeReply();
- reply->setResult(api::ReturnCode(api::ReturnCode::BUSY));
- _distributor->handleReply(std::shared_ptr<api::StorageReply>(std::move(reply)));
-
- auto& node_info = pending_message_tracker().getNodeInfo();
-
- EXPECT_TRUE(node_info.isBusy(0));
- getClock().addSecondsToTime(99);
- EXPECT_TRUE(node_info.isBusy(0));
- getClock().addSecondsToTime(2);
- EXPECT_FALSE(node_info.isBusy(0));
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, external_client_requests_are_handled_individually_in_priority_order) {
- setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1");
- addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a");
-
- std::vector<api::StorageMessage::Priority> priorities({50, 255, 10, 40, 0});
- document::DocumentId id("id:foo:testdoctype1:n=1:foo");
- vespalib::stringref field_set = "";
- for (auto pri : priorities) {
- auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(document::BucketId()), id, field_set);
- cmd->setPriority(pri);
- // onDown appends to internal message FIFO queue, awaiting hand-off.
- _distributor->onDown(cmd);
- }
- // At the hand-off point we expect client requests to be prioritized.
- // For each tick, a priority-order client request is processed and sent off.
- for (size_t i = 1; i <= priorities.size(); ++i) {
- tickDistributorNTimes(1);
- ASSERT_EQ(i, _sender.commands().size());
- }
-
- std::vector<int> expected({0, 10, 40, 50, 255});
- std::vector<int> actual;
- for (auto& msg : _sender.commands()) {
- actual.emplace_back(static_cast<int>(msg->getPriority()));
- }
- EXPECT_THAT(actual, ContainerEq(expected));
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, internal_messages_are_started_in_fifo_order_batch) {
- // To test internal request ordering, we use NotifyBucketChangeCommand
- // for the reason that it explicitly updates the bucket database for
- // each individual invocation.
- setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1");
- document::BucketId bucket(16, 1);
- addNodesToBucketDB(bucket, "0=1/1/1/t");
-
- std::vector<api::StorageMessage::Priority> priorities({50, 255, 10, 40, 1});
- for (auto pri : priorities) {
- api::BucketInfo fake_info(pri, pri, pri);
- auto cmd = std::make_shared<api::NotifyBucketChangeCommand>(makeDocumentBucket(bucket), fake_info);
- cmd->setSourceIndex(0);
- cmd->setPriority(pri);
- _distributor->onDown(cmd);
- }
-
- // Doing a single tick should process all internal requests in one batch
- tickDistributorNTimes(1);
- ASSERT_EQ(5, _sender.replies().size());
-
- // The bucket info for priority 1 (last FIFO-order change command received, but
- // highest priority) should be the end-state of the bucket database, _not_ that
- // of lowest priority 255.
- BucketDatabase::Entry e(getBucket(bucket));
- EXPECT_EQ(api::BucketInfo(1, 1, 1), e.getBucketInfo().getNode(0)->getBucketInfo());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, closing_aborts_priority_queued_client_requests) {
- setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1");
- document::BucketId bucket(16, 1);
- addNodesToBucketDB(bucket, "0=1/1/1/t");
-
- document::DocumentId id("id:foo:testdoctype1:n=1:foo");
- vespalib::stringref field_set = "";
- for (int i = 0; i < 10; ++i) {
- auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(document::BucketId()), id, field_set);
- _distributor->onDown(cmd);
- }
- tickDistributorNTimes(1);
- // Closing should trigger 1 abort via startet GetOperation and 9 aborts from pri queue
- _distributor->close();
- ASSERT_EQ(10, _sender.replies().size());
- for (auto& msg : _sender.replies()) {
- EXPECT_EQ(api::ReturnCode::ABORTED, dynamic_cast<api::StorageReply&>(*msg).getResult().getResult());
- }
-}
-
-namespace {
-
-void assert_invalid_stats_for_all_spaces(
- const BucketSpacesStatsProvider::PerNodeBucketSpacesStats& stats,
- uint16_t node_index) {
- auto stats_iter = stats.find(node_index);
- ASSERT_TRUE(stats_iter != stats.cend());
- ASSERT_EQ(2, stats_iter->second.size());
- auto space_iter = stats_iter->second.find(document::FixedBucketSpaces::default_space_name());
- ASSERT_TRUE(space_iter != stats_iter->second.cend());
- ASSERT_FALSE(space_iter->second.valid());
- space_iter = stats_iter->second.find(document::FixedBucketSpaces::global_space_name());
- ASSERT_TRUE(space_iter != stats_iter->second.cend());
- ASSERT_FALSE(space_iter->second.valid());
-}
-
-}
-
-// Migrated to DistributorStripeTest
-// Cross-stripe bucket stats test added in TopLevelDistributorTest::entering_recovery_mode_resets_bucket_space_stats_across_all_stripes
-TEST_F(LegacyDistributorTest, entering_recovery_mode_resets_bucket_space_stats) {
- // Set up a cluster state + DB contents which implies merge maintenance ops
- setupDistributor(Redundancy(2), NodeCount(2), "version:1 distributor:1 storage:2");
- addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a");
- addNodesToBucketDB(document::BucketId(16, 2), "0=1/1/1/t/a");
- addNodesToBucketDB(document::BucketId(16, 3), "0=2/2/2/t/a");
-
- tickDistributorNTimes(5); // 1/3rds into second round through database
-
- enableDistributorClusterState("version:2 distributor:1 storage:3 .1.s:d");
- EXPECT_TRUE(distributor_is_in_recovery_mode());
- // Bucket space stats should now be invalid per space per node, pending stats
- // from state version 2. Exposing stats from version 1 risks reporting stale
- // information back to the cluster controller.
- const auto stats = distributor_bucket_spaces_stats();
- ASSERT_EQ(2, stats.size());
-
- assert_invalid_stats_for_all_spaces(stats, 0);
- assert_invalid_stats_for_all_spaces(stats, 2);
-}
-
-// Migrated to TopLevelDistributorTest
-TEST_F(LegacyDistributorTest, leaving_recovery_mode_immediately_sends_getnodestate_replies) {
- setupDistributor(Redundancy(2), NodeCount(2), "version:1 distributor:1 storage:2");
- // Should not send explicit replies during init stage
- ASSERT_EQ(0, explicit_node_state_reply_send_invocations());
- // Add a couple of buckets so we have something to iterate over
- addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a");
- addNodesToBucketDB(document::BucketId(16, 2), "0=1/1/1/t/a");
-
- enableDistributorClusterState("version:2 distributor:1 storage:3 .1.s:d");
- EXPECT_TRUE(distributor_is_in_recovery_mode());
- EXPECT_EQ(0, explicit_node_state_reply_send_invocations());
- tickDistributorNTimes(1); // DB round not yet complete
- EXPECT_EQ(0, explicit_node_state_reply_send_invocations());
- tickDistributorNTimes(2); // DB round complete after 2nd bucket + "scan done" discovery tick
- EXPECT_EQ(1, explicit_node_state_reply_send_invocations());
- EXPECT_FALSE(distributor_is_in_recovery_mode());
- // Now out of recovery mode, subsequent round completions should not send replies
- tickDistributorNTimes(10);
- EXPECT_EQ(1, explicit_node_state_reply_send_invocations());
-}
-
-void LegacyDistributorTest::do_test_pending_merge_getnodestate_reply_edge(BucketSpace space) {
- setupDistributor(Redundancy(2), NodeCount(2), "version:1 distributor:1 storage:2");
- EXPECT_TRUE(distributor_is_in_recovery_mode());
- // 2 buckets with missing replicas triggering merge pending stats
- addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a");
- addNodesToBucketDB(Bucket(space, BucketId(16, 2)), "0=1/1/1/t/a");
- tickDistributorNTimes(3);
- EXPECT_FALSE(distributor_is_in_recovery_mode());
- const auto space_name = FixedBucketSpaces::to_string(space);
- assertBucketSpaceStats(2, 0, 1, space_name, _distributor->getBucketSpacesStats());
- // First completed scan sends off merge stats et al to cluster controller
- EXPECT_EQ(1, explicit_node_state_reply_send_invocations());
-
- // Edge not triggered when 1 bucket with missing replica left
- addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a,1=1/1/1/t");
- tickDistributorNTimes(3);
- assertBucketSpaceStats(1, 1, 1, space_name, _distributor->getBucketSpacesStats());
- EXPECT_EQ(1, explicit_node_state_reply_send_invocations());
-
- // Edge triggered when no more buckets with requiring merge
- addNodesToBucketDB(Bucket(space, BucketId(16, 2)), "0=1/1/1/t/a,1=1/1/1/t");
- tickDistributorNTimes(3);
- assertBucketSpaceStats(0, 2, 1, space_name, _distributor->getBucketSpacesStats());
- EXPECT_EQ(2, explicit_node_state_reply_send_invocations());
-
- // Should only send when edge happens, not in subsequent DB iterations
- tickDistributorNTimes(10);
- EXPECT_EQ(2, explicit_node_state_reply_send_invocations());
-
- // Going back to merges pending should _not_ send a getnodestate reply (at least for now)
- addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a");
- tickDistributorNTimes(3);
- assertBucketSpaceStats(1, 1, 1, space_name, _distributor->getBucketSpacesStats());
- EXPECT_EQ(2, explicit_node_state_reply_send_invocations());
-}
-
-// TODO: rewrite into DistributorStripeTest
-TEST_F(LegacyDistributorTest, pending_to_no_pending_default_merges_edge_immediately_sends_getnodestate_replies) {
- do_test_pending_merge_getnodestate_reply_edge(FixedBucketSpaces::default_space());
-}
-
-// TODO: rewrite into DistributorStripeTest
-TEST_F(LegacyDistributorTest, pending_to_no_pending_global_merges_edge_immediately_sends_getnodestate_replies) {
- do_test_pending_merge_getnodestate_reply_edge(FixedBucketSpaces::global_space());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, stale_reads_config_is_propagated_to_external_operation_handler) {
- createLinks();
- setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
-
- configure_stale_reads_enabled(true);
- EXPECT_TRUE(getExternalOperationHandler().concurrent_gets_enabled());
-
- configure_stale_reads_enabled(false);
- EXPECT_FALSE(getExternalOperationHandler().concurrent_gets_enabled());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, fast_path_on_consistent_gets_config_is_propagated_to_internal_config) {
- createLinks();
- setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
-
- configure_update_fast_path_restart_enabled(true);
- EXPECT_TRUE(getConfig().update_fast_path_restart_enabled());
-
- configure_update_fast_path_restart_enabled(false);
- EXPECT_FALSE(getConfig().update_fast_path_restart_enabled());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, merge_disabling_config_is_propagated_to_internal_config) {
- createLinks();
- setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
-
- configure_merge_operations_disabled(true);
- EXPECT_TRUE(getConfig().merge_operations_disabled());
-
- configure_merge_operations_disabled(false);
- EXPECT_FALSE(getConfig().merge_operations_disabled());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, metadata_update_phase_config_is_propagated_to_internal_config) {
- createLinks();
- setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
-
- configure_metadata_update_phase_enabled(true);
- EXPECT_TRUE(getConfig().enable_metadata_only_fetch_phase_for_inconsistent_updates());
-
- configure_metadata_update_phase_enabled(false);
- EXPECT_FALSE(getConfig().enable_metadata_only_fetch_phase_for_inconsistent_updates());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, weak_internal_read_consistency_config_is_propagated_to_internal_configs) {
- createLinks();
- setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
-
- configure_use_weak_internal_read_consistency(true);
- EXPECT_TRUE(getConfig().use_weak_internal_read_consistency_for_client_gets());
- EXPECT_TRUE(getExternalOperationHandler().use_weak_internal_read_consistency_for_gets());
-
- configure_use_weak_internal_read_consistency(false);
- EXPECT_FALSE(getConfig().use_weak_internal_read_consistency_for_client_gets());
- EXPECT_FALSE(getExternalOperationHandler().use_weak_internal_read_consistency_for_gets());
-}
-
-void LegacyDistributorTest::set_up_and_start_get_op_with_stale_reads_enabled(bool enabled) {
- createLinks();
- setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
- configure_stale_reads_enabled(enabled);
-
- document::BucketId bucket(16, 1);
- addNodesToBucketDB(bucket, "0=1/1/1/t");
- _distributor->onDown(make_dummy_get_command_for_bucket_1());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, gets_are_started_outside_main_distributor_logic_if_stale_reads_enabled) {
- set_up_and_start_get_op_with_stale_reads_enabled(true);
- ASSERT_THAT(_sender.commands(), SizeIs(1));
- EXPECT_THAT(_sender.replies(), SizeIs(0));
-
- // Reply is routed to the correct owner
- auto reply = std::shared_ptr<api::StorageReply>(_sender.command(0)->makeReply());
- _distributor->onDown(reply);
- ASSERT_THAT(_sender.commands(), SizeIs(1));
- EXPECT_THAT(_sender.replies(), SizeIs(1));
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, gets_are_not_started_outside_main_distributor_logic_if_stale_reads_disabled) {
- set_up_and_start_get_op_with_stale_reads_enabled(false);
- // Get has been placed into distributor queue, so no external messages are produced.
- EXPECT_THAT(_sender.commands(), SizeIs(0));
- EXPECT_THAT(_sender.replies(), SizeIs(0));
-}
-
-// Migrated to DistributorStripeTest
-// There's no need or desire to track "lockfree" Gets in the main pending message tracker,
-// as we only have to track mutations to inhibit maintenance ops safely. Furthermore,
-// the message tracker is a multi-index and therefore has some runtime cost.
-TEST_F(LegacyDistributorTest, gets_started_outside_main_thread_are_not_tracked_by_main_pending_message_tracker) {
- set_up_and_start_get_op_with_stale_reads_enabled(true);
- Bucket bucket(FixedBucketSpaces::default_space(), BucketId(16, 1));
- EXPECT_FALSE(pending_message_tracker().hasPendingMessage(
- 0, bucket, api::MessageType::GET_ID));
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, closing_aborts_gets_started_outside_main_distributor_thread) {
- set_up_and_start_get_op_with_stale_reads_enabled(true);
- _distributor->close();
- ASSERT_EQ(1, _sender.replies().size());
- EXPECT_EQ(api::ReturnCode::ABORTED, _sender.reply(0)->getResult().getResult());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, prioritize_global_bucket_merges_config_is_propagated_to_internal_config) {
- createLinks();
- setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
-
- configure_prioritize_global_bucket_merges(true);
- EXPECT_TRUE(getConfig().prioritize_global_bucket_merges());
-
- configure_prioritize_global_bucket_merges(false);
- EXPECT_FALSE(getConfig().prioritize_global_bucket_merges());
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, max_activation_inhibited_out_of_sync_groups_config_is_propagated_to_internal_config) {
- createLinks();
- setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
-
- configure_max_activation_inhibited_out_of_sync_groups(3);
- EXPECT_EQ(getConfig().max_activation_inhibited_out_of_sync_groups(), 3);
-
- configure_max_activation_inhibited_out_of_sync_groups(0);
- EXPECT_EQ(getConfig().max_activation_inhibited_out_of_sync_groups(), 0);
-}
-
-// Migrated to DistributorStripeTest
-TEST_F(LegacyDistributorTest, wanted_split_bit_count_is_lower_bounded) {
- createLinks();
- setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
-
- ConfigBuilder builder;
- builder.minsplitcount = 7;
- configureDistributor(builder);
-
- EXPECT_EQ(getConfig().getMinimalBucketSplit(), 8);
-}
-
-// Migrated to TopLevelDistributorTest
-TEST_F(LegacyDistributorTest, host_info_sent_immediately_once_all_stripes_first_reported) {
- set_num_distributor_stripes(4);
- createLinks();
- getClock().setAbsoluteTimeInSeconds(1000);
- // TODO STRIPE can't call this currently since it touches the bucket DB updater directly:
- // setupDistributor(Redundancy(2), NodeCount(2), "version:1 distributor:1 storage:2");
-
- tickDistributorNTimes(1);
- EXPECT_EQ(0, explicit_node_state_reply_send_invocations()); // Nothing yet
- getDistributor().notify_stripe_wants_to_send_host_info(1);
- getDistributor().notify_stripe_wants_to_send_host_info(2);
- getDistributor().notify_stripe_wants_to_send_host_info(3);
-
- tickDistributorNTimes(1);
- // Still nothing. Missing initial report from stripe 0
- EXPECT_EQ(0, explicit_node_state_reply_send_invocations());
-
- getDistributor().notify_stripe_wants_to_send_host_info(0);
- tickDistributorNTimes(1);
- // All stripes have reported in, it's time to party!
- EXPECT_EQ(1, explicit_node_state_reply_send_invocations());
-
- // No further sends if stripes haven't requested it yet.
- getClock().setAbsoluteTimeInSeconds(2000);
- tickDistributorNTimes(10);
- EXPECT_EQ(1, explicit_node_state_reply_send_invocations());
-}
-
-// Migrated to TopLevelDistributorTest
-TEST_F(LegacyDistributorTest, non_bootstrap_host_info_send_request_delays_sending) {
- set_num_distributor_stripes(4);
- createLinks();
- getClock().setAbsoluteTimeInSeconds(1000);
-
- for (uint16_t i = 0; i < 4; ++i) {
- getDistributor().notify_stripe_wants_to_send_host_info(i);
- }
- tickDistributorNTimes(1);
- // Bootstrap case
- EXPECT_EQ(1, explicit_node_state_reply_send_invocations());
-
- // Stripe 1 suddenly really wants to tell the cluster controller something again
- getDistributor().notify_stripe_wants_to_send_host_info(1);
- tickDistributorNTimes(1);
- // But its cry for attention is not yet honored since the delay hasn't passed.
- EXPECT_EQ(1, explicit_node_state_reply_send_invocations());
-
- getClock().addMilliSecondsToTime(999);
- tickDistributorNTimes(1);
- // 1 sec delay has still not passed
- EXPECT_EQ(1, explicit_node_state_reply_send_invocations());
-
- getClock().addMilliSecondsToTime(1);
- tickDistributorNTimes(1);
- // But now it has
- EXPECT_EQ(2, explicit_node_state_reply_send_invocations());
-}
-
-}
diff --git a/storage/src/tests/distributor/statusreporterdelegatetest.cpp b/storage/src/tests/distributor/statusreporterdelegatetest.cpp
index 9e66f1920e2..3cac901619e 100644
--- a/storage/src/tests/distributor/statusreporterdelegatetest.cpp
+++ b/storage/src/tests/distributor/statusreporterdelegatetest.cpp
@@ -1,7 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include <tests/common/testhelper.h>
-#include <tests/distributor/distributortestutil.h>
+#include <tests/common/teststorageapp.h>
#include <vespa/storage/distributor/statusreporterdelegate.h>
#include <vespa/vespalib/gtest/gtest.h>
diff --git a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp
index 01f7d5a4f0a..d57ce228908 100644
--- a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp
+++ b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp
@@ -630,7 +630,7 @@ TopLevelBucketDBUpdaterTest::trigger_completed_but_not_yet_activated_transition(
}
TEST_F(TopLevelBucketDBUpdaterTest, normal_usage) {
- set_cluster_state(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"));
+ set_cluster_state(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); // FIXME init mode why?
ASSERT_EQ(message_count(3), _sender.commands().size());
@@ -638,21 +638,21 @@ TEST_F(TopLevelBucketDBUpdaterTest, normal_usage) {
ASSERT_EQ(_component->getDistribution()->getNodeGraph().getDistributionConfigHash(),
dynamic_cast<const RequestBucketInfoCommand&>(*_sender.command(0)).getDistributionHash());
- ASSERT_NO_FATAL_FAILURE(fake_bucket_reply(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"),
+ ASSERT_NO_FATAL_FAILURE(fake_bucket_reply(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), // FIXME init mode why?
*_sender.command(0), 10));
_sender.clear();
// Optimization for not refetching unneeded data after cluster state
// change is only implemented after completion of previous cluster state
- set_cluster_state("distributor:2 .0.s:i storage:3");
+ set_cluster_state("distributor:2 .0.s:i storage:3"); // FIXME init mode why?
ASSERT_EQ(message_count(3), _sender.commands().size());
// Expect reply of first set SystemState request.
ASSERT_EQ(size_t(1), _sender.replies().size());
ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(
- lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"),
+ lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), // FIXME init mode why?
message_count(3), 10));
ASSERT_NO_FATAL_FAILURE(assert_correct_buckets(10, "distributor:2 storage:3"));
}
@@ -661,9 +661,9 @@ TEST_F(TopLevelBucketDBUpdaterTest, distributor_change) {
int num_buckets = 100;
// First sends request
- set_cluster_state("distributor:2 .0.s:i .1.s:i storage:3");
+ set_cluster_state("distributor:2 .0.s:i .1.s:i storage:3"); // FIXME init mode why?
ASSERT_EQ(message_count(3), _sender.commands().size());
- ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"),
+ ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), // FIXME init mode why?
message_count(3), num_buckets));
_sender.clear();
@@ -718,14 +718,14 @@ TEST_F(TopLevelBucketDBUpdaterTest, distributor_change_with_grouping) {
}
TEST_F(TopLevelBucketDBUpdaterTest, normal_usage_initializing) {
- set_cluster_state("distributor:1 .0.s:i storage:1 .0.s:i");
+ set_cluster_state("distributor:1 .0.s:i storage:1 .0.s:i"); // FIXME init mode why?
ASSERT_EQ(_bucket_spaces.size(), _sender.commands().size());
// Not yet passing on system state.
ASSERT_EQ(size_t(0), _sender_down.commands().size());
- ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 .0.s:i storage:1"),
+ ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), // FIXME init mode why?
_bucket_spaces.size(), 10, 10));
ASSERT_NO_FATAL_FAILURE(assert_correct_buckets(10, "distributor:1 storage:1"));
@@ -740,12 +740,12 @@ TEST_F(TopLevelBucketDBUpdaterTest, normal_usage_initializing) {
_sender.clear();
_sender_down.clear();
- set_cluster_state("distributor:1 .0.s:i storage:1");
+ set_cluster_state("distributor:1 .0.s:i storage:1"); // FIXME init mode why?
// Send a new request bucket info up.
ASSERT_EQ(_bucket_spaces.size(), _sender.commands().size());
- ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 .0.s:i storage:1"),
+ ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), // FIXME init mode why?
_bucket_spaces.size(), 20));
// Pass on cluster state and recheck buckets now.
@@ -755,7 +755,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, normal_usage_initializing) {
}
TEST_F(TopLevelBucketDBUpdaterTest, failed_request_bucket_info) {
- set_cluster_state("distributor:1 .0.s:i storage:1");
+ set_cluster_state("distributor:1 .0.s:i storage:1"); // FIXME init mode why?
// 2 messages sent up: 1 to the nodes, and one reply to the setsystemstate.
ASSERT_EQ(_bucket_spaces.size(), _sender.commands().size());
@@ -781,7 +781,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, failed_request_bucket_info) {
ASSERT_EQ(size_t(0), _sender_down.commands().size());
for (uint32_t i = 0; i < _bucket_spaces.size(); ++i) {
- ASSERT_NO_FATAL_FAILURE(fake_bucket_reply(lib::ClusterState("distributor:1 .0.s:i storage:1"),
+ ASSERT_NO_FATAL_FAILURE(fake_bucket_reply(lib::ClusterState("distributor:1 .0.s:i storage:1"), // FIXME init mode why?
*_sender.command(_bucket_spaces.size() + i), 10));
}
diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.cpp b/storage/src/tests/distributor/top_level_distributor_test_util.cpp
index b6e9beb38ae..e4002b3d1cb 100644
--- a/storage/src/tests/distributor/top_level_distributor_test_util.cpp
+++ b/storage/src/tests/distributor/top_level_distributor_test_util.cpp
@@ -303,11 +303,6 @@ TopLevelDistributorTestUtil::total_distributor_metrics() const
return *_distributor->_total_metrics;
}
-const storage::distributor::DistributorNodeContext&
-TopLevelDistributorTestUtil::node_context() const {
- return _distributor->distributor_component();
-}
-
DistributorBucketSpace&
TopLevelDistributorTestUtil::distributor_bucket_space(const document::BucketId& id)
{
diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.h b/storage/src/tests/distributor/top_level_distributor_test_util.h
index 8832f8ada6e..e8794eb4199 100644
--- a/storage/src/tests/distributor/top_level_distributor_test_util.h
+++ b/storage/src/tests/distributor/top_level_distributor_test_util.h
@@ -69,7 +69,6 @@ public:
TopLevelBucketDBUpdater& bucket_db_updater();
const IdealStateMetricSet& total_ideal_state_metrics() const;
const DistributorMetricSet& total_distributor_metrics() const;
- const storage::distributor::DistributorNodeContext& node_context() const;
DistributorBucketSpace& distributor_bucket_space(const document::BucketId& id);
const DistributorBucketSpace& distributor_bucket_space(const document::BucketId& id) const;