summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2023-08-10 14:23:48 +0000
committerHenning Baldersheim <balder@yahoo-inc.com>2023-08-10 16:19:51 +0000
commit81badc222c0bff453af4c7cd0529c701c8f2d591 (patch)
tree42930d2ddbfe306572560f8681b60481f0478313
parentfdcb755aff2d29773914f50b1ce6cde59deb7956 (diff)
Reduce use of default values in method calls
-rw-r--r--storage/src/tests/distributor/distributor_stripe_test_util.cpp100
-rw-r--r--storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp70
-rw-r--r--storage/src/tests/distributor/top_level_distributor_test_util.cpp11
-rw-r--r--storage/src/vespa/storage/distributor/activecopy.cpp5
-rw-r--r--storage/src/vespa/storage/storageutil/distributorstatecache.h8
-rw-r--r--storage/src/vespa/storage/tools/getidealstate.cpp15
-rw-r--r--vdslib/src/tests/distribution/distributiontest.cpp123
-rw-r--r--vdslib/src/vespa/vdslib/distribution/distribution.cpp2
-rw-r--r--vdslib/src/vespa/vdslib/distribution/distribution.h49
9 files changed, 123 insertions, 260 deletions
diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.cpp b/storage/src/tests/distributor/distributor_stripe_test_util.cpp
index 7a64eda28ff..6ececa39583 100644
--- a/storage/src/tests/distributor/distributor_stripe_test_util.cpp
+++ b/storage/src/tests/distributor/distributor_stripe_test_util.cpp
@@ -40,34 +40,22 @@ DistributorStripeTestUtil::createLinks()
_node = std::make_unique<TestDistributorApp>(_config.getConfigId());
_metrics = std::make_shared<DistributorMetricSet>();
_ideal_state_metrics = std::make_shared<IdealStateMetricSet>();
- _stripe = std::make_unique<DistributorStripe>(_node->getComponentRegister(),
- *_metrics,
- *_ideal_state_metrics,
- _node->node_identity(),
- _messageSender,
- *this,
- _done_initializing);
+ _stripe = std::make_unique<DistributorStripe>(_node->getComponentRegister(), *_metrics, *_ideal_state_metrics,
+ _node->node_identity(), _messageSender, *this, _done_initializing);
}
void
-DistributorStripeTestUtil::setup_stripe(int redundancy,
- int nodeCount,
- const std::string& systemState,
- uint32_t earlyReturn,
- bool requirePrimaryToBeWritten)
+DistributorStripeTestUtil::setup_stripe(int redundancy, int nodeCount, const std::string& systemState,
+ uint32_t earlyReturn, bool requirePrimaryToBeWritten)
{
setup_stripe(redundancy, nodeCount, lib::ClusterStateBundle(lib::ClusterState(systemState)), earlyReturn, requirePrimaryToBeWritten);
}
void
-DistributorStripeTestUtil::setup_stripe(int redundancy,
- int node_count,
- const lib::ClusterStateBundle& state,
- uint32_t early_return,
- bool require_primary_to_be_written)
+DistributorStripeTestUtil::setup_stripe(int redundancy, int node_count, const lib::ClusterStateBundle& state,
+ uint32_t early_return, bool require_primary_to_be_written)
{
- lib::Distribution::DistributionConfigBuilder config(
- lib::Distribution::getDefaultDistributionConfig(redundancy, node_count).get());
+ lib::Distribution::DistributionConfigBuilder config(lib::Distribution::getDefaultDistributionConfig(redundancy, node_count).get());
config.redundancy = redundancy;
config.initialRedundancy = early_return;
config.ensurePrimaryPersisted = require_primary_to_be_written;
@@ -93,8 +81,7 @@ DistributorStripeTestUtil::setup_stripe(int redundancy,
void
DistributorStripeTestUtil::set_redundancy(uint32_t redundancy)
{
- auto distribution = std::make_shared<lib::Distribution>(
- lib::Distribution::getDefaultDistributionConfig(redundancy, 100));
+ auto distribution = std::make_shared<lib::Distribution>(lib::Distribution::getDefaultDistributionConfig(redundancy, 100));
// Same rationale for not triggering a full distribution change as
// in setup_stripe() above
_node->getComponentRegister().setDistribution(distribution);
@@ -217,8 +204,7 @@ DistributorStripeTestUtil::getIdealStr(document::BucketId id, const lib::Cluster
}
std::vector<uint16_t> nodes;
- getDistribution().getIdealNodes(
- lib::NodeType::STORAGE, state, id, nodes);
+ getDistribution().getIdealNodes(lib::NodeType::STORAGE, state, id, nodes, "uim");
std::sort(nodes.begin(), nodes.end());
std::ostringstream ost;
ost << id << ": " << dumpVector(nodes);
@@ -226,8 +212,7 @@ DistributorStripeTestUtil::getIdealStr(document::BucketId id, const lib::Cluster
}
void
-DistributorStripeTestUtil::addIdealNodes(const lib::ClusterState& state,
- const document::BucketId& id)
+DistributorStripeTestUtil::addIdealNodes(const lib::ClusterState& state, const document::BucketId& id)
{
BucketDatabase::Entry entry = getBucket(id);
@@ -236,15 +221,11 @@ DistributorStripeTestUtil::addIdealNodes(const lib::ClusterState& state,
}
std::vector<uint16_t> res;
- getDistribution().getIdealNodes(
- lib::NodeType::STORAGE, state, id, res);
+ getDistribution().getIdealNodes(lib::NodeType::STORAGE, state, id, res, "uim");
for (uint32_t i = 0; i < res.size(); ++i) {
- if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() !=
- lib::State::MAINTENANCE)
- {
- entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)),
- toVector<uint16_t>(0));
+ if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() != lib::State::MAINTENANCE) {
+ entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)), toVector<uint16_t>(0));
}
}
@@ -292,10 +273,7 @@ DistributorStripeTestUtil::addNodesToBucketDB(const document::Bucket& bucket, co
}
uint16_t idx = atoi(tok2[0].data());
- BucketCopy node(
- 0,
- idx,
- info);
+ BucketCopy node(0, idx, info);
// Allow user to manually override trusted and active.
if (tok3.size() > flagsIdx && tok3[flagsIdx] == "t") {
@@ -309,44 +287,32 @@ DistributorStripeTestUtil::addNodesToBucketDB(const document::Bucket& bucket, co
}
void
-DistributorStripeTestUtil::addNodesToBucketDB(const document::BucketId& id,
- const std::string& nodeStr)
-{
+DistributorStripeTestUtil::addNodesToBucketDB(const document::BucketId& id, const std::string& nodeStr) {
addNodesToBucketDB(document::Bucket(makeBucketSpace(), id), nodeStr);
}
void
-DistributorStripeTestUtil::removeFromBucketDB(const document::BucketId& id)
-{
+DistributorStripeTestUtil::removeFromBucketDB(const document::BucketId& id) {
getBucketDatabase().remove(id);
}
void
-DistributorStripeTestUtil::addIdealNodes(const document::BucketId& id)
-{
+DistributorStripeTestUtil::addIdealNodes(const document::BucketId& id) {
// TODO STRIPE roundabout way of getting state bundle..!
addIdealNodes(*operation_context().cluster_state_bundle().getBaselineClusterState(), id);
}
void
-DistributorStripeTestUtil::insertBucketInfo(document::BucketId id,
- uint16_t node,
- uint32_t checksum,
- uint32_t count,
- uint32_t size,
- bool trusted,
- bool active)
+DistributorStripeTestUtil::insertBucketInfo(document::BucketId id, uint16_t node, uint32_t checksum,
+ uint32_t count, uint32_t size, bool trusted, bool active)
{
api::BucketInfo info(checksum, count, size);
insertBucketInfo(id, node, info, trusted, active);
}
void
-DistributorStripeTestUtil::insertBucketInfo(document::BucketId id,
- uint16_t node,
- const api::BucketInfo& info,
- bool trusted,
- bool active)
+DistributorStripeTestUtil::insertBucketInfo(document::BucketId id, uint16_t node, const api::BucketInfo& info,
+ bool trusted, bool active)
{
BucketDatabase::Entry entry = getBucketDatabase().get(id);
if (!entry.valid()) {
@@ -358,9 +324,7 @@ DistributorStripeTestUtil::insertBucketInfo(document::BucketId id,
info2.setActive();
}
BucketCopy copy(operation_context().generate_unique_timestamp(), node, info2);
-
entry->addNode(copy.setTrusted(trusted), toVector<uint16_t>(0));
-
getBucketDatabase().update(entry);
}
@@ -371,9 +335,7 @@ DistributorStripeTestUtil::dumpBucket(const document::BucketId& bid)
}
void
-DistributorStripeTestUtil::sendReply(Operation& op,
- int idx,
- api::ReturnCode::Result result)
+DistributorStripeTestUtil::sendReply(Operation& op, int idx, api::ReturnCode::Result result)
{
if (idx == -1) {
idx = _sender.commands().size() - 1;
@@ -387,20 +349,17 @@ DistributorStripeTestUtil::sendReply(Operation& op,
}
BucketDatabase::Entry
-DistributorStripeTestUtil::getBucket(const document::Bucket& bucket) const
-{
+DistributorStripeTestUtil::getBucket(const document::Bucket& bucket) const {
return getBucketDatabase(bucket.getBucketSpace()).get(bucket.getBucketId());
}
BucketDatabase::Entry
-DistributorStripeTestUtil::getBucket(const document::BucketId& bId) const
-{
+DistributorStripeTestUtil::getBucket(const document::BucketId& bId) const {
return getBucketDatabase().get(bId);
}
void
-DistributorStripeTestUtil::disableBucketActivationInConfig(bool disable)
-{
+DistributorStripeTestUtil::disableBucketActivationInConfig(bool disable) {
ConfigBuilder builder;
builder.disableBucketActivation = disable;
configure_stripe(builder);
@@ -437,14 +396,12 @@ DistributorStripeTestUtil::doc_selection_parser() const {
}
DistributorMetricSet&
-DistributorStripeTestUtil::metrics()
-{
+DistributorStripeTestUtil::metrics() {
return *_metrics;
}
bool
-DistributorStripeTestUtil::tick()
-{
+DistributorStripeTestUtil::tick() {
return _stripe->tick();
}
@@ -553,8 +510,7 @@ DistributorStripeTestUtil::getBucketSpaces() const
void
DistributorStripeTestUtil::enable_cluster_state(vespalib::stringref state)
{
- getBucketDBUpdater().simulate_cluster_state_bundle_activation(
- lib::ClusterStateBundle(lib::ClusterState(state)));
+ getBucketDBUpdater().simulate_cluster_state_bundle_activation(lib::ClusterStateBundle(lib::ClusterState(state)));
}
void
diff --git a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp
index 567e0a947da..7eb9dfe6269 100644
--- a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp
+++ b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp
@@ -65,12 +65,9 @@ public:
close();
}
- std::shared_ptr<RequestBucketInfoReply> make_fake_bucket_reply(
- const lib::ClusterState& state,
- const RequestBucketInfoCommand& cmd,
- int storageIndex,
- uint32_t bucketCount,
- uint32_t invalidBucketCount = 0)
+ std::shared_ptr<RequestBucketInfoReply>
+ make_fake_bucket_reply(const lib::ClusterState& state, const RequestBucketInfoCommand& cmd,
+ int storageIndex, uint32_t bucketCount,uint32_t invalidBucketCount = 0)
{
auto sreply = std::make_shared<RequestBucketInfoReply>(cmd);
sreply->setAddress(storage_address(storageIndex));
@@ -84,19 +81,14 @@ public:
}
std::vector<uint16_t> nodes;
- distributor_bucket_space(bucket).getDistribution().getIdealNodes(
- lib::NodeType::STORAGE, state, bucket, nodes);
+ distributor_bucket_space(bucket).getDistribution().getIdealNodes(lib::NodeType::STORAGE, state, bucket, nodes, "uim");
for (uint32_t j = 0; j < nodes.size(); ++j) {
if (nodes[j] == storageIndex) {
if (i >= bucketCount) {
- vec.push_back(api::RequestBucketInfoReply::Entry(
- document::BucketId(16, i),
- api::BucketInfo()));
+ vec.emplace_back(document::BucketId(16, i), api::BucketInfo());
} else {
- vec.push_back(api::RequestBucketInfoReply::Entry(
- document::BucketId(16, i),
- api::BucketInfo(10,1,1)));
+ vec.emplace_back(document::BucketId(16, i), api::BucketInfo(10,1,1));
}
}
}
@@ -105,45 +97,34 @@ public:
return sreply;
}
- void fake_bucket_reply(const lib::ClusterState &state,
- const api::StorageCommand &cmd,
- uint32_t bucket_count,
- uint32_t invalid_bucket_count = 0)
+ void fake_bucket_reply(const lib::ClusterState &state, const api::StorageCommand &cmd,
+ uint32_t bucket_count, uint32_t invalid_bucket_count = 0)
{
ASSERT_EQ(cmd.getType(), MessageType::REQUESTBUCKETINFO);
const api::StorageMessageAddress& address(*cmd.getAddress());
bucket_db_updater().onRequestBucketInfoReply(
- make_fake_bucket_reply(state,
- dynamic_cast<const RequestBucketInfoCommand &>(cmd),
- address.getIndex(),
- bucket_count,
- invalid_bucket_count));
+ make_fake_bucket_reply(state, dynamic_cast<const RequestBucketInfoCommand &>(cmd),
+ address.getIndex(), bucket_count, invalid_bucket_count));
}
- void fake_bucket_reply(const lib::ClusterState &state,
- const api::StorageCommand &cmd,
- uint32_t bucket_count,
+ void fake_bucket_reply(const lib::ClusterState &state, const api::StorageCommand &cmd, uint32_t bucket_count,
const std::function<void(api::RequestBucketInfoReply&)>& reply_decorator)
{
ASSERT_EQ(cmd.getType(), MessageType::REQUESTBUCKETINFO);
const api::StorageMessageAddress& address(*cmd.getAddress());
- auto reply = make_fake_bucket_reply(state,
- dynamic_cast<const RequestBucketInfoCommand &>(cmd),
- address.getIndex(),
- bucket_count, 0);
+ auto reply = make_fake_bucket_reply(state, dynamic_cast<const RequestBucketInfoCommand &>(cmd),
+ address.getIndex(), bucket_count, 0);
reply_decorator(*reply);
bucket_db_updater().onRequestBucketInfoReply(reply);
}
- void send_fake_reply_for_single_bucket_request(
- const api::RequestBucketInfoCommand& rbi)
+ void send_fake_reply_for_single_bucket_request(const api::RequestBucketInfoCommand& rbi)
{
ASSERT_EQ(size_t(1), rbi.getBuckets().size());
const document::BucketId& bucket(rbi.getBuckets()[0]);
auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi);
- reply->getBucketInfo().push_back(
- api::RequestBucketInfoReply::Entry(bucket, api::BucketInfo(20, 10, 12, 50, 60, true, true)));
+ reply->getBucketInfo().emplace_back(bucket, api::BucketInfo(20, 10, 12, 50, 60, true, true));
stripe_of_bucket(bucket).bucket_db_updater().onRequestBucketInfoReply(reply);
}
@@ -154,15 +135,11 @@ public:
}
std::vector<uint16_t> nodes;
- distributor_bucket_space(id).getDistribution().getIdealNodes(
- lib::NodeType::STORAGE, state, document::BucketId(id), nodes);
+ distributor_bucket_space(id).getDistribution().getIdealNodes(lib::NodeType::STORAGE, state, document::BucketId(id), nodes, "uim");
if (nodes.size() != entry->getNodeCount()) {
- return vespalib::make_string("Bucket Id %s has %d nodes in "
- "ideal state, but has only %d in DB",
- id.toString().c_str(),
- (int)nodes.size(),
- (int)entry->getNodeCount());
+ return vespalib::make_string("Bucket Id %s has %d nodes in ideal state, but has only %d in DB",
+ id.toString().c_str(), (int)nodes.size(), (int)entry->getNodeCount());
}
for (uint32_t i = 0; i<nodes.size(); i++) {
@@ -175,10 +152,7 @@ public:
}
if (!found) {
- return vespalib::make_string(
- "Bucket Id %s has no copy from node %d",
- id.toString().c_str(),
- nodes[i]);
+ return vespalib::make_string("Bucket Id %s has no copy from node %d", id.toString().c_str(), nodes[i]);
}
}
@@ -188,13 +162,11 @@ public:
struct OrderByIncreasingNodeIndex {
template <typename T>
bool operator()(const T& lhs, const T& rhs) {
- return (lhs->getAddress()->getIndex()
- < rhs->getAddress()->getIndex());
+ return (lhs->getAddress()->getIndex() < rhs->getAddress()->getIndex());
}
};
- void sort_sent_messages_by_index(DistributorMessageSenderStub& sender,
- size_t sortFromOffset = 0)
+ void sort_sent_messages_by_index(DistributorMessageSenderStub& sender, size_t sortFromOffset = 0)
{
std::sort(sender.commands().begin() + sortFromOffset,
sender.commands().end(),
diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.cpp b/storage/src/tests/distributor/top_level_distributor_test_util.cpp
index 9677ea568e8..9859a6fb237 100644
--- a/storage/src/tests/distributor/top_level_distributor_test_util.cpp
+++ b/storage/src/tests/distributor/top_level_distributor_test_util.cpp
@@ -187,7 +187,7 @@ TopLevelDistributorTestUtil::get_ideal_str(document::BucketId id, const lib::Clu
return id.toString();
}
std::vector<uint16_t> nodes;
- _component->getDistribution()->getIdealNodes(lib::NodeType::STORAGE, state, id, nodes);
+ _component->getDistribution()->getIdealNodes(lib::NodeType::STORAGE, state, id, nodes, "uim");
std::sort(nodes.begin(), nodes.end());
std::ostringstream ost;
ost << id << ": " << dumpVector(nodes);
@@ -205,14 +205,11 @@ TopLevelDistributorTestUtil::add_ideal_nodes(const lib::ClusterState& state, con
std::vector<uint16_t> res;
assert(_component.get());
- _component->getDistribution()->getIdealNodes(lib::NodeType::STORAGE, state, id, res);
+ _component->getDistribution()->getIdealNodes(lib::NodeType::STORAGE, state, id, res, "uim");
for (uint32_t i = 0; i < res.size(); ++i) {
- if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() !=
- lib::State::MAINTENANCE)
- {
- entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)),
- toVector<uint16_t>(0));
+ if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() != lib::State::MAINTENANCE) {
+ entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)), toVector<uint16_t>(0));
}
}
diff --git a/storage/src/vespa/storage/distributor/activecopy.cpp b/storage/src/vespa/storage/distributor/activecopy.cpp
index 57052c9c509..ea194e1be21 100644
--- a/storage/src/vespa/storage/distributor/activecopy.cpp
+++ b/storage/src/vespa/storage/distributor/activecopy.cpp
@@ -144,10 +144,9 @@ ActiveCopy::calculate(const std::vector<uint16_t>& idealState, const lib::Distri
if (validNodesWithCopy.empty()) {
return ActiveList();
}
- using IndexList = std::vector<uint16_t>;
- std::vector<IndexList> groups;
+ std::vector<lib::Distribution::IndexList> groups;
if (distribution.activePerGroup()) {
- groups = distribution.splitNodesIntoLeafGroups(std::move(validNodesWithCopy));
+ groups = distribution.splitNodesIntoLeafGroups(validNodesWithCopy);
} else {
groups.push_back(std::move(validNodesWithCopy));
}
diff --git a/storage/src/vespa/storage/storageutil/distributorstatecache.h b/storage/src/vespa/storage/storageutil/distributorstatecache.h
index 8c4d07e39bf..0652a980e3a 100644
--- a/storage/src/vespa/storage/storageutil/distributorstatecache.h
+++ b/storage/src/vespa/storage/storageutil/distributorstatecache.h
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
+
#include <vespa/vdslib/state/clusterstate.h>
#include <vespa/vdslib/distribution/distribution.h>
@@ -9,9 +10,7 @@ namespace storage {
class DistributorStateCache
{
public:
- DistributorStateCache(
- const lib::Distribution& distr,
- const lib::ClusterState& state)
+ DistributorStateCache(const lib::Distribution& distr, const lib::ClusterState& state)
: _distribution(distr),
_state(state),
_distrBitMask(0xffffffffffffffffull),
@@ -22,8 +21,7 @@ public:
_distrBitMask >>= (64 - state.getDistributionBitCount());
}
- uint16_t getOwner(const document::BucketId& bid,
- const char* upStates = "ui")
+ uint16_t getOwner(const document::BucketId& bid, const char* upStates = "ui")
{
uint64_t distributionBits = bid.getRawId() & _distrBitMask;
diff --git a/storage/src/vespa/storage/tools/getidealstate.cpp b/storage/src/vespa/storage/tools/getidealstate.cpp
index 8b120924aaa..9e80517f4f7 100644
--- a/storage/src/vespa/storage/tools/getidealstate.cpp
+++ b/storage/src/vespa/storage/tools/getidealstate.cpp
@@ -64,18 +64,13 @@ Options::Options(int argc, const char* const* argv)
Options::~Options() {}
-void processBucket(const lib::Distribution& distribution,
- const lib::ClusterState& clusterState,
- const std::string& upStates,
- const document::BucketId& bucket)
+void processBucket(const lib::Distribution& distribution, const lib::ClusterState& clusterState,
+ const std::string& upStates, const document::BucketId& bucket)
{
std::ostringstream ost;
- std::vector<uint16_t> storageNodes(distribution.getIdealStorageNodes(
- clusterState, bucket, upStates.c_str()));
- uint16_t distributorNode(distribution.getIdealDistributorNode(
- clusterState, bucket, upStates.c_str()));
- ost << bucket << " distributor: " << distributorNode
- << ", storage:";
+ std::vector<uint16_t> storageNodes(distribution.getIdealStorageNodes(clusterState, bucket, upStates.c_str()));
+ uint16_t distributorNode(distribution.getIdealDistributorNode(clusterState, bucket, upStates.c_str()));
+ ost << bucket << " distributor: " << distributorNode << ", storage:";
for (uint32_t i=0; i<storageNodes.size(); ++i) {
ost << " " << storageNodes[i];
}
diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp
index b5c756aece9..33a6d47b719 100644
--- a/vdslib/src/tests/distribution/distributiontest.cpp
+++ b/vdslib/src/tests/distribution/distributiontest.cpp
@@ -53,9 +53,7 @@ TEST(DistributionTest, test_verify_java_distributions)
long maxBucket = 1;
long mask = 0;
- for (uint32_t distributionBits = 0; distributionBits <= 32;
- ++distributionBits)
- {
+ for (uint32_t distributionBits = 0; distributionBits <= 32; ++distributionBits) {
state.setDistributionBitCount(distributionBits);
RandomGen randomizer(distributionBits);
for (uint32_t bucketIndex = 0; bucketIndex < 64; ++bucketIndex) {
@@ -66,11 +64,8 @@ TEST(DistributionTest, test_verify_java_distributions)
bucketId = randomizer.nextUint64();
}
document::BucketId bucket(distributionBits, bucketId);
- for (uint32_t redundancy = 1;
- redundancy <= distr.getRedundancy(); ++redundancy)
- {
- int distributorIndex = distr.getIdealDistributorNode(
- state, bucket, "uim");
+ for (uint32_t redundancy = 1; redundancy <= distr.getRedundancy(); ++redundancy) {
+ int distributorIndex = distr.getIdealDistributorNode(state, bucket, "uim");
of << distributionBits << " " << (bucketId & mask)
<< " " << redundancy << " " << distributorIndex << "\n";
}
@@ -102,22 +97,16 @@ struct ExpectedResult {
};
void
-verifyJavaDistribution(const vespalib::string& name,
- const ClusterState& state,
- const Distribution& distribution,
- const NodeType& nodeType,
- uint16_t redundancy,
- uint16_t nodeCount,
- vespalib::stringref upStates,
- const std::vector<ExpectedResult> results)
+verifyJavaDistribution(const vespalib::string& name, const ClusterState& state, const Distribution& distribution,
+ const NodeType& nodeType, uint16_t redundancy, uint16_t nodeCount,
+ vespalib::stringref upStates, const std::vector<ExpectedResult> results)
{
(void) nodeCount;
for (uint32_t i=0, n=results.size(); i<n; ++i) {
std::string testId = name + " " + results[i].bucket.toString();
try {
std::vector<uint16_t> nvect;
- distribution.getIdealNodes(nodeType, state, results[i].bucket,
- nvect, upStates.data(), redundancy);
+ distribution.getIdealNodes(nodeType, state, results[i].bucket, nvect, upStates.data(), redundancy);
IdealNodeList nodes;
for (uint32_t j=0, m=nvect.size(); j<m; ++j) {
nodes.push_back(Node(nodeType, nvect[j]));
@@ -155,8 +144,7 @@ auto readFile(const std::string & filename) {
TEST(DistributionTest, test_verify_java_distributions_2)
{
- vespalib::DirectoryList files(
- vespalib::listDirectory("distribution/testdata"));
+ vespalib::DirectoryList files(vespalib::listDirectory("distribution/testdata"));
for (uint32_t i=0, n=files.size(); i<n; ++i) {
size_t pos = files[i].find(".java.results");
if (pos == vespalib::string::npos || pos + 13 != files[i].size()) {
@@ -189,8 +177,7 @@ TEST(DistributionTest, test_verify_java_distributions_2)
ClusterState cs(c["cluster-state"].asString().make_string());
std::string distConfig(c["distribution"].asString().make_string());
Distribution d(distConfig);
- const NodeType& nt(
- NodeType::get(c["node-type"].asString().make_string()));
+ const NodeType& nt(NodeType::get(c["node-type"].asString().make_string()));
uint32_t redundancy(c["redundancy"].asLong());
uint32_t nodeCount(c["node-count"].asLong());
vespalib::string upStates(c["up-states"].asString().make_string());
@@ -209,8 +196,7 @@ TEST(DistributionTest, test_verify_java_distributions_2)
}
results.push_back(result);
}
- verifyJavaDistribution(name, cs, d, nt, redundancy, nodeCount,
- upStates, results);
+ verifyJavaDistribution(name, cs, d, nt, redundancy, nodeCount, upStates, results);
//std::cerr << name << ": Verified " << results.size() << " tests.\n";
}
}
@@ -223,8 +209,7 @@ TEST(DistributionTest, test_unchanged_distribution)
std::ifstream in("distribution/testdata/41-distributordistribution");
for (unsigned i = 0; i < 64_Ki; i++) {
- uint16_t node = distr.getIdealDistributorNode(
- state, document::BucketId(16, i), "u");
+ uint16_t node = distr.getIdealDistributorNode(state, document::BucketId(16, i), "u");
char buf[100];
in.getline(buf, 100);
@@ -272,9 +257,7 @@ struct MyTest {
document::BucketId bucket(16, i);
std::vector<uint16_t> nodes;
ClusterState clusterState(_state);
- _distribution->getIdealNodes(
- *_nodeType, clusterState, bucket, nodes,
- _upStates, _redundancy);
+ _distribution->getIdealNodes(*_nodeType, clusterState, bucket, nodes, _upStates, _redundancy);
for (uint32_t j=0; j<nodes.size(); ++j) {
++result[nodes[j]];
}
@@ -293,8 +276,7 @@ MyTest::MyTest()
{ }
MyTest::~MyTest() = default;
-std::vector<uint16_t> createNodeCountList(const std::string& source,
- std::vector<uint16_t>& vals) {
+std::vector<uint16_t> createNodeCountList(const std::string& source, std::vector<uint16_t>& vals) {
std::vector<uint16_t> result(vals.size(), 0);
vespalib::StringTokenizer st(source, " ");
for (uint32_t i=0; i<st.size(); ++i) {
@@ -375,15 +357,9 @@ TEST(DistributionTest, testHighSplitBit)
document::BucketId bid1 = document::BucketId(bits, base);
document::BucketId bid2 = document::BucketId(bits, base);
- std::vector<uint16_t> nodes1 =
- distr.getIdealStorageNodes(state,
- bid1,
- "u");
+ std::vector<uint16_t> nodes1 = distr.getIdealStorageNodes(state, bid1, "u");
- std::vector<uint16_t> nodes2 =
- distr.getIdealStorageNodes(state,
- bid2,
- "u");
+ std::vector<uint16_t> nodes2 = distr.getIdealStorageNodes(state, bid2, "u");
ost1 << bid1 << " vs. " << bid2 << ": ";
ost2 << bid1 << " vs. " << bid2 << ": ";
@@ -424,16 +400,14 @@ TEST(DistributionTest, test_distribution)
s1 << "storage:" << n << std::endl;
ClusterState systemState(s1.str());
- Distribution distr(
- Distribution::getDefaultDistributionConfig(3, n));
+ Distribution distr(Distribution::getDefaultDistributionConfig(3, n));
std::vector<std::pair<uint64_t, std::vector<uint16_t> > > _distribution(b);
std::vector<int> _nodeCount(n, 0);
for (int i = 0; i < b; i++) {
_distribution[i].first = i;
- _distribution[i].second = distr.getIdealStorageNodes(
- systemState, document::BucketId(26, i));
+ _distribution[i].second = distr.getIdealStorageNodes(systemState, document::BucketId(26, i));
sort(_distribution[i].second.begin(), _distribution[i].second.end());
auto unique_nodes = std::distance(_distribution[i].second.begin(), unique(_distribution[i].second.begin(), _distribution[i].second.end()));
_distribution[i].second.resize(unique_nodes);
@@ -469,9 +443,7 @@ TEST(DistributionTest, test_move)
{
ClusterState systemState("storage:3");
document::BucketId bucket(16, 0x8b4f67ae);
-
Distribution distr(Distribution::getDefaultDistributionConfig(2, 3));
-
res = distr.getIdealStorageNodes(systemState, bucket);
EXPECT_EQ(size_t(2), res.size());
}
@@ -479,11 +451,8 @@ TEST(DistributionTest, test_move)
std::vector<uint16_t> res2;
{
ClusterState systemState("storage:4");
-
Distribution distr(Distribution::getDefaultDistributionConfig(2, 4));
-
document::BucketId bucket(16, 0x8b4f67ae);
-
res2 = distr.getIdealStorageNodes(systemState, bucket);
EXPECT_EQ(size_t(2), res2.size());
}
@@ -506,8 +475,7 @@ TEST(DistributionTest, test_move_constraints)
std::vector<std::vector<uint16_t> > initBuckets(10000);
for (unsigned i = 0; i < initBuckets.size(); i++) {
- initBuckets[i] = distr.getIdealStorageNodes(
- clusterState, document::BucketId(16, i));
+ initBuckets[i] = distr.getIdealStorageNodes(clusterState, document::BucketId(16, i));
sort(initBuckets[i].begin(), initBuckets[i].end());
}
@@ -517,8 +485,7 @@ TEST(DistributionTest, test_move_constraints)
ClusterState systemState("storage:11 .10.s:d");
for (unsigned i = 0; i < addedDownBuckets.size(); i++) {
- addedDownBuckets[i] = distr.getIdealStorageNodes(
- systemState, document::BucketId(16, i));
+ addedDownBuckets[i] = distr.getIdealStorageNodes(systemState, document::BucketId(16, i));
sort(addedDownBuckets[i].begin(), addedDownBuckets[i].end());
}
for (unsigned i = 0; i < initBuckets.size(); i++) {
@@ -541,15 +508,14 @@ TEST(DistributionTest, test_move_constraints)
ClusterState systemState("storage:10 .0.s:d");
for (unsigned i = 0; i < removed0Buckets.size(); i++) {
- removed0Buckets[i] = distr.getIdealStorageNodes(
- systemState, document::BucketId(16, i));
+ removed0Buckets[i] = distr.getIdealStorageNodes(systemState, document::BucketId(16, i));
sort(removed0Buckets[i].begin(), removed0Buckets[i].end());
}
for (unsigned i = 0; i < initBuckets.size(); i++) {
std::vector<uint16_t> movedAway;
set_difference(initBuckets[i].begin(), initBuckets[i].end(),
- removed0Buckets[i].begin(), removed0Buckets[i].end(),
- back_inserter(movedAway));
+ removed0Buckets[i].begin(), removed0Buckets[i].end(),
+ back_inserter(movedAway));
if (movedAway.size() > 0) {
if (movedAway[0] != 0) {
std::cerr << i << ": ";
@@ -572,15 +538,14 @@ TEST(DistributionTest, test_move_constraints)
ClusterState systemState("storage:11");
for (unsigned i = 0; i < added10Buckets.size(); i++) {
- added10Buckets[i] = distr.getIdealStorageNodes(
- systemState, document::BucketId(16, i));
+ added10Buckets[i] = distr.getIdealStorageNodes(systemState, document::BucketId(16, i));
sort(added10Buckets[i].begin(), added10Buckets[i].end());
}
for (unsigned i = 0; i < initBuckets.size(); i++) {
std::vector<uint16_t> movedInto;
std::set_difference(added10Buckets[i].begin(), added10Buckets[i].end(),
- initBuckets[i].begin(), initBuckets[i].end(),
- std::inserter(movedInto, movedInto.begin()));
+ initBuckets[i].begin(), initBuckets[i].end(),
+ std::inserter(movedInto, movedInto.begin()));
if (movedInto.size() > 0) {
ASSERT_EQ((size_t)1, movedInto.size());
EXPECT_EQ((uint16_t)10, movedInto[0]);
@@ -601,11 +566,9 @@ TEST(DistributionTest, test_distribution_bits)
for (unsigned i = 0; i < 100; i++) {
int val = rand();
- uint32_t index = distr.getIdealDistributorNode(
- state1, document::BucketId(19, val), "u");
+ uint32_t index = distr.getIdealDistributorNode(state1, document::BucketId(19, val), "u");
ost1 << index << " ";
- index = distr.getIdealDistributorNode(
- state2, document::BucketId(19, val), "u");
+ index = distr.getIdealDistributorNode(state2, document::BucketId(19, val), "u");
ost2 << index << " ";
}
@@ -620,10 +583,8 @@ TEST(DistributionTest, test_redundancy_hierarchical_distribution)
Distribution distr2(Distribution::getDefaultDistributionConfig(2, 10));
for (unsigned i = 0; i < 100; i++) {
- uint16_t d1 = distr1.getIdealDistributorNode(
- state, document::BucketId(16, i), "u");
- uint16_t d2 = distr2.getIdealDistributorNode(
- state, document::BucketId(16, i), "u");
+ uint16_t d1 = distr1.getIdealDistributorNode(state, document::BucketId(16, i), "u");
+ uint16_t d2 = distr2.getIdealDistributorNode(state, document::BucketId(16, i), "u");
EXPECT_EQ(d1, d2);
}
}
@@ -653,20 +614,17 @@ TEST(DistributionTest, test_hierarchical_distribution)
ClusterState state("distributor:6 storage:6");
for (uint32_t i = 0; i < 3; ++i) {
- EXPECT_EQ(
- vespalib::string("rack0"),
- distr.getNodeGraph().getGroupForNode(i)->getName());
+ EXPECT_EQ(vespalib::string("rack0"),
+ distr.getNodeGraph().getGroupForNode(i)->getName());
}
for (uint32_t i = 3; i < 6; ++i) {
- EXPECT_EQ(
- vespalib::string("rack1"),
- distr.getNodeGraph().getGroupForNode(i)->getName());
+ EXPECT_EQ(vespalib::string("rack1"),
+ distr.getNodeGraph().getGroupForNode(i)->getName());
}
std::vector<int> mainNode(6);
for (uint32_t i=0; i<100; ++i) {
- std::vector<uint16_t> nodes = distr.getIdealStorageNodes(
- state, document::BucketId(16, i), "u");
+ std::vector<uint16_t> nodes = distr.getIdealStorageNodes(state, document::BucketId(16, i), "u");
ASSERT_EQ((size_t) 4, nodes.size());
EXPECT_LT(nodes[0], mainNode.size());
++mainNode[nodes[0]];
@@ -710,8 +668,7 @@ TEST(DistributionTest, test_group_capacity)
int group0count = 0;
int group1count = 0;
for (uint32_t i = 0; i < 1000; i++) {
- std::vector<uint16_t> nodes = distr.getIdealStorageNodes(
- state, document::BucketId(16, i), "u");
+ std::vector<uint16_t> nodes = distr.getIdealStorageNodes(state, document::BucketId(16, i), "u");
if (nodes[0] == 0 || nodes[0] == 1 || nodes[0] == 2) {
group0count++;
}
@@ -794,14 +751,12 @@ TEST(DistributionTest, test_hierarchical_no_redistribution)
EXPECT_EQ(numBuckets, v.size());
v.clear();
- state.setNodeState(Node(NodeType::STORAGE, 0),
- NodeState(NodeType::STORAGE, State::DOWN));
+ state.setNodeState(Node(NodeType::STORAGE, 0),NodeState(NodeType::STORAGE, State::DOWN));
std::vector< std::vector<uint16_t> > distr2(4);
for (size_t i = 0; i < numBuckets; i++) {
- nodes = distribution.getIdealStorageNodes(
- state, document::BucketId(16, i), "u");
+ nodes = distribution.getIdealStorageNodes(state, document::BucketId(16, i), "u");
for (uint16_t j=0; j<nodes.size(); ++j) {
ASSERT_TRUE(0 != nodes[j]);
distr2[nodes[j]].push_back(i);
@@ -1010,7 +965,7 @@ group[2].nodes[1].retired false
auto nodes_of = [&](uint32_t bucket){
std::vector<uint16_t> actual;
- distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, bucket), actual);
+ distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, bucket), actual, "uim");
return actual;
};
@@ -1071,7 +1026,7 @@ TEST(DistributionTest, DISABLED_benchmark_ideal_state_for_many_groups) {
std::vector<uint16_t> actual;
uint32_t bucket = 0;
auto min_time = vespalib::BenchmarkTimer::benchmark([&]{
- distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, (bucket++ & 0xffffU)), actual);
+ distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, (bucket++ & 0xffffU)), actual, "uim");
}, 5.0);
fprintf(stderr, "%.10f seconds\n", min_time);
}
diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.cpp b/vdslib/src/vespa/vdslib/distribution/distribution.cpp
index e732b9faccd..87a1f6d3758 100644
--- a/vdslib/src/vespa/vdslib/distribution/distribution.cpp
+++ b/vdslib/src/vespa/vdslib/distribution/distribution.cpp
@@ -467,7 +467,7 @@ Distribution::getIdealDistributorNode(const ClusterState& state, const document:
}
std::vector<Distribution::IndexList>
-Distribution::splitNodesIntoLeafGroups(IndexList nodeList) const
+Distribution::splitNodesIntoLeafGroups(vespalib::ConstArrayRef<uint16_t> nodeList) const
{
std::vector<IndexList> result;
std::map<uint16_t, IndexList> nodes;
diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.h b/vdslib/src/vespa/vdslib/distribution/distribution.h
index 79c31e83c49..b39afb17e15 100644
--- a/vdslib/src/vespa/vdslib/distribution/distribution.h
+++ b/vdslib/src/vespa/vdslib/distribution/distribution.h
@@ -12,6 +12,7 @@
#include <vespa/document/bucket/bucketid.h>
#include <vespa/vdslib/state/nodetype.h>
#include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/util/arrayref.h>
namespace vespa::config::content::internal {
class InternalStorDistributionType;
@@ -38,9 +39,9 @@ private:
uint16_t _redundancy;
uint16_t _initialRedundancy;
uint16_t _readyCopies;
- bool _activePerGroup;
- bool _ensurePrimaryPersisted;
- bool _distributorAutoOwnershipTransferOnWholeGroupDown;
+ bool _activePerGroup;
+ bool _ensurePrimaryPersisted;
+ bool _distributorAutoOwnershipTransferOnWholeGroupDown;
vespalib::string _serialized;
struct ResultGroup {
@@ -105,33 +106,26 @@ public:
Distribution& operator=(const Distribution&) = delete;
- const vespalib::string& serialize() const { return _serialized; }
+ const vespalib::string& serialize() const noexcept { return _serialized; }
- const Group& getNodeGraph() const { return *_nodeGraph; }
- uint16_t getRedundancy() const { return _redundancy; }
- uint16_t getInitialRedundancy() const { return _initialRedundancy; }
- uint16_t getReadyCopies() const { return _readyCopies; }
- bool ensurePrimaryPersisted() const { return _ensurePrimaryPersisted; }
- bool distributorAutoOwnershipTransferOnWholeGroupDown() const
- { return _distributorAutoOwnershipTransferOnWholeGroupDown; }
- bool activePerGroup() const { return _activePerGroup; }
+ const Group& getNodeGraph() const noexcept { return *_nodeGraph; }
+ uint16_t getRedundancy() const noexcept { return _redundancy; }
+ uint16_t getInitialRedundancy() const noexcept { return _initialRedundancy; }
+ uint16_t getReadyCopies() const noexcept { return _readyCopies; }
+ bool ensurePrimaryPersisted() const noexcept { return _ensurePrimaryPersisted; }
+ bool distributorAutoOwnershipTransferOnWholeGroupDown() const noexcept { return _distributorAutoOwnershipTransferOnWholeGroupDown; }
+ bool activePerGroup() const noexcept { return _activePerGroup; }
- bool operator==(const Distribution& o) const
- { return (_serialized == o._serialized); }
- bool operator!=(const Distribution& o) const
- { return (_serialized != o._serialized); }
+ bool operator==(const Distribution& o) const noexcept { return (_serialized == o._serialized); }
+ bool operator!=(const Distribution& o) const noexcept { return (_serialized != o._serialized); }
void print(std::ostream& out, bool, const std::string&) const override;
/** Simplified wrapper for getIdealNodes() */
- std::vector<uint16_t> getIdealStorageNodes(
- const ClusterState&, const document::BucketId&,
- const char* upStates = "uim") const;
+ std::vector<uint16_t> getIdealStorageNodes(const ClusterState&, const document::BucketId&, const char* upStates = "uim") const;
/** Simplified wrapper for getIdealNodes() */
- uint16_t getIdealDistributorNode(
- const ClusterState&, const document::BucketId&,
- const char* upStates = "uim") const;
+ uint16_t getIdealDistributorNode(const ClusterState&, const document::BucketId&, const char* upStates = "uim") const;
/**
* @throws TooFewBucketBitsInUseException If distribution bit count is
@@ -140,25 +134,22 @@ public:
* in any upstate.
*/
enum { DEFAULT_REDUNDANCY = 0xffff };
- void getIdealNodes(const NodeType&, const ClusterState&,
- const document::BucketId&, std::vector<uint16_t>& nodes,
- const char* upStates = "uim",
- uint16_t redundancy = DEFAULT_REDUNDANCY) const;
+ void getIdealNodes(const NodeType&, const ClusterState&, const document::BucketId&, std::vector<uint16_t>& nodes,
+ const char* upStates, uint16_t redundancy = DEFAULT_REDUNDANCY) const;
/**
* Unit tests can use this function to get raw config for this class to use
* with a really simple setup with no hierarchical grouping. This function
* should not be used by any production code.
*/
- static ConfigWrapper getDefaultDistributionConfig(
- uint16_t redundancy = 2, uint16_t nodeCount = 10);
+ static ConfigWrapper getDefaultDistributionConfig(uint16_t redundancy = 2, uint16_t nodeCount = 10);
/**
* Utility function used by distributor to split copies into groups to
* handle active per group feature.
*/
using IndexList = std::vector<uint16_t>;
- std::vector<IndexList> splitNodesIntoLeafGroups(IndexList nodes) const;
+ std::vector<IndexList> splitNodesIntoLeafGroups(vespalib::ConstArrayRef<uint16_t> nodes) const;
static bool allDistributorsDown(const Group&, const ClusterState&);
};