diff options
Diffstat (limited to 'vdslib')
8 files changed, 78 insertions, 53 deletions
diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index e90f2c25de3..d037c3f14b1 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/distribution/idealnodecalculator.h> #include <vespa/config/helper/configfetcher.h> @@ -17,6 +16,7 @@ #include <vespa/vespalib/stllike/lexical_cast.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/config-stor-distribution.h> namespace storage { namespace lib { @@ -580,8 +580,7 @@ DistributionTest::testDiskCapacityWeights() void DistributionTest::testDiskSkewLocal() { - Distribution distr(Distribution::getDefaultDistributionConfig( - 2, 3, vespa::config::content::StorDistributionConfig::MODULO_INDEX)); + Distribution distr(Distribution::getDefaultDistributionConfig(2, 3, Distribution::MODULO_INDEX)); std::vector<float> diskDist(100); NodeState nodeState; nodeState.setDiskCount(100); @@ -597,17 +596,13 @@ DistributionTest::testDiskSkewLocal() } - - void DistributionTest::testDiskSkewGlobal() { uint16_t num_disks = 10; uint16_t num_nodes = 10; - Distribution distr(Distribution::getDefaultDistributionConfig( - 2, num_nodes, vespa::config::content::StorDistributionConfig::MODULO_INDEX)); - std::vector<std::vector<float> > diskDist( - num_nodes, std::vector<float>(num_disks)); + Distribution distr(Distribution::getDefaultDistributionConfig(2, num_nodes, Distribution::MODULO_INDEX)); + std::vector<std::vector<float> > diskDist(num_nodes, std::vector<float>(num_disks)); NodeState nodeState; nodeState.setDiskCount(num_disks); for(uint16_t idx=0; idx < num_nodes; idx++) { @@ -640,8 +635,7 @@ DistributionTest::testDiskIntersection() uint16_t num_disks = 8; uint16_t num_nodes = 20; float max = 0; - Distribution distr(Distribution::getDefaultDistributionConfig( - 2, num_nodes, vespa::config::content::StorDistributionConfig::MODULO_INDEX)); + Distribution distr(Distribution::getDefaultDistributionConfig(2, num_nodes, Distribution::MODULO_INDEX)); NodeState nodeState; nodeState.setDiskCount(num_disks); diff --git a/vdslib/src/tests/distribution/idealnodecalculatorimpltest.cpp b/vdslib/src/tests/distribution/idealnodecalculatorimpltest.cpp index dcecf935e1f..44683e0481e 100644 --- a/vdslib/src/tests/distribution/idealnodecalculatorimpltest.cpp +++ b/vdslib/src/tests/distribution/idealnodecalculatorimpltest.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vdslib/distribution/idealnodecalculatorimpl.h> +#include <vespa/config-stor-distribution.h> #include <vespa/vdstestlib/cppunit/macros.h> namespace storage { diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.cpp b/vdslib/src/vespa/vdslib/distribution/distribution.cpp index da3fa2f3302..4bc0a3d09c6 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/distribution.cpp @@ -1,14 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vdslib/distribution/distribution.h> - -#include <cmath> -#include <vespa/document/bucket/bucketid.h> +#include "distribution.h" #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/util/bobhash.h> -#include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/stllike/asciistream.h> #include <boost/lexical_cast.hpp> #include <vespa/config/config.h> @@ -16,6 +11,7 @@ #include <vespa/config/print/asciiconfigwriter.h> #include <vespa/config/print/asciiconfigreader.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/config-stor-distribution.h> #include <vespa/log/log.h> LOG_SETUP(".vdslib.distribution"); @@ -108,6 +104,8 @@ Distribution::operator=(const Distribution& d) return *this; } +Distribution::~Distribution() { } + namespace { std::vector<uint16_t> getGroupPath(const vespalib::stringref & path) { vespalib::StringTokenizer st(path, ".", ""); @@ -119,6 +117,28 @@ namespace { } } +namespace { + using ConfigDiskDistribution = vespa::config::content::StorDistributionConfig::DiskDistribution; + Distribution::DiskDistribution fromConfig(ConfigDiskDistribution cfg) { + switch (cfg) { + case ConfigDiskDistribution::MODULO : return Distribution::MODULO; + case ConfigDiskDistribution::MODULO_BID : return Distribution::MODULO_BID; + case ConfigDiskDistribution::MODULO_INDEX : return Distribution::MODULO_INDEX; + case ConfigDiskDistribution::MODULO_KNUTH : return Distribution::MODULO_KNUTH; + } + abort(); + } + ConfigDiskDistribution toConfig(Distribution::DiskDistribution cfg) { + switch (cfg) { + case Distribution::MODULO : return ConfigDiskDistribution::MODULO; + case Distribution::MODULO_BID : return ConfigDiskDistribution::MODULO_BID; + case Distribution::MODULO_INDEX : return ConfigDiskDistribution::MODULO_INDEX; + case Distribution::MODULO_KNUTH : return ConfigDiskDistribution::MODULO_KNUTH; + } + abort(); + } +} + void Distribution::configure(const vespa::config::content::StorDistributionConfig& config) { @@ -170,7 +190,7 @@ Distribution::configure(const vespa::config::content::StorDistributionConfig& co _redundancy = config.redundancy; _initialRedundancy = config.initialRedundancy; _ensurePrimaryPersisted = config.ensurePrimaryPersisted; - _diskDistribution = config.diskDistribution; + _diskDistribution = fromConfig(config.diskDistribution); _readyCopies = config.readyCopies; _activePerGroup = config.activePerLeafGroup; _distributorAutoOwnershipTransferOnWholeGroupDown @@ -253,9 +273,13 @@ Distribution::getDiskSeed( } } throw vespalib::IllegalStateException("Unknown disk distribution: " - + Config::getDiskDistributionName(_diskDistribution), VESPA_STRLOC); + + Config::getDiskDistributionName(toConfig(_diskDistribution)), VESPA_STRLOC); } +void +Distribution::print(std::ostream& out, bool, const std::string&) const { + out << serialize(); +} // This function should only depend on disk distribution and node index. It is // assumed that any other change, for instance in hierarchical grouping, does @@ -603,9 +627,7 @@ Distribution::getIdealNodes(const NodeType& nodeType, } Distribution::DistributionConfig -Distribution::getDefaultDistributionConfig( - uint16_t redundancy, uint16_t nodeCount, - DiskDistribution distr) +Distribution::getDefaultDistributionConfig(uint16_t redundancy, uint16_t nodeCount, DiskDistribution distr) { vespa::config::content::StorDistributionConfigBuilder config; config.redundancy = redundancy; @@ -617,7 +639,7 @@ Distribution::getDefaultDistributionConfig( for (uint16_t i=0; i<nodeCount; ++i) { config.group[0].nodes[i].index = i; } - config.diskDistribution = distr; + config.diskDistribution = toConfig(distr); return config; } diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.h b/vdslib/src/vespa/vdslib/distribution/distribution.h index 263bf403550..59bc1610d7a 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.h +++ b/vdslib/src/vespa/vdslib/distribution/distribution.h @@ -11,9 +11,17 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/vdslib/distribution/group.h> #include <vespa/vdslib/state/nodetype.h> -#include <vespa/config-stor-distribution.h> #include <vespa/vespalib/util/exception.h> +namespace vespa { + namespace config { + namespace content { + namespace internal { + class InternalStorDistributionType; + } + } + } +} namespace storage { namespace lib { @@ -27,9 +35,9 @@ class Distribution : public document::Printable { public: typedef std::shared_ptr<Distribution> SP; typedef std::unique_ptr<Distribution> UP; - typedef vespa::config::content::StorDistributionConfigBuilder - DistributionConfig; - typedef DistributionConfig::DiskDistribution DiskDistribution; + using DistributionConfig = const vespa::config::content::internal::InternalStorDistributionType; + using DistributionConfigBuilder = vespa::config::content::internal::InternalStorDistributionType; + enum DiskDistribution { MODULO, MODULO_INDEX, MODULO_KNUTH, MODULO_BID }; private: std::vector<uint32_t> _distributionBitMasks; @@ -102,13 +110,14 @@ private: * You need to create a new distribution object to change it. This function * is thus private so only constructor can call it. */ - void configure(const vespa::config::content::StorDistributionConfig & config); + void configure(const DistributionConfig & config); public: Distribution(); Distribution(const Distribution&); Distribution(const DistributionConfig&); Distribution(const vespalib::string& serialized); + ~Distribution(); Distribution& operator=(const Distribution&); @@ -129,18 +138,7 @@ public: bool operator!=(const Distribution& o) const { return (_serialized != o._serialized); } - void print(std::ostream& out, bool, const std::string&) const - { out << serialize(); } - - /** - * Used by bucket position mapper to do a "thread safe" update. Just - * changing the primitive _diskDistribution value should be safe as long as - * it doesn't matter if it takes some time before the changed value is - * seen. (Should remove this once bucket position mapper can be safely - * reconfigured while not in use) - */ - void setDiskDistribution(DistributionConfig::DiskDistribution d) - { _diskDistribution = d; } + void print(std::ostream& out, bool, const std::string&) const override; enum DISK_MODE { IDEAL_DISK_EVEN_IF_DOWN, @@ -182,7 +180,7 @@ public: */ static DistributionConfig getDefaultDistributionConfig( uint16_t redundancy = 2, uint16_t nodeCount = 10, - DiskDistribution distr = DistributionConfig::MODULO_BID); + DiskDistribution distr = MODULO_BID); /** * Utility function used by distributor to split copies into groups to diff --git a/vdslib/src/vespa/vdslib/distribution/group.cpp b/vdslib/src/vespa/vdslib/distribution/group.cpp index 7a988d58121..b1a81c85dbb 100644 --- a/vdslib/src/vespa/vdslib/distribution/group.cpp +++ b/vdslib/src/vespa/vdslib/distribution/group.cpp @@ -5,6 +5,7 @@ #include <vespa/vdslib/state/random.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <ostream> namespace storage { namespace lib { diff --git a/vdslib/src/vespa/vdslib/distribution/group.h b/vdslib/src/vespa/vdslib/distribution/group.h index 38249999352..8e8740ca739 100644 --- a/vdslib/src/vespa/vdslib/distribution/group.h +++ b/vdslib/src/vespa/vdslib/distribution/group.h @@ -14,7 +14,6 @@ #include <map> #include <vector> #include <vespa/vespalib/objects/floatingpointtype.h> -#include <vespa/config-stor-distribution.h> #include <vespa/vdslib/distribution/redundancygroupdistribution.h> #include <vespa/vespalib/util/crc.h> diff --git a/vdslib/src/vespa/vdslib/distribution/idealnodecalculator.h b/vdslib/src/vespa/vdslib/distribution/idealnodecalculator.h index 2f43907801e..b1acc62a946 100644 --- a/vdslib/src/vespa/vdslib/distribution/idealnodecalculator.h +++ b/vdslib/src/vespa/vdslib/distribution/idealnodecalculator.h @@ -22,7 +22,8 @@ class IdealNodeList : public document::Printable { std::vector<Node> _idealNodes; public: - IdealNodeList() : _idealNodes() {} + IdealNodeList(); + ~IdealNodeList(); void push_back(const Node& node) { _idealNodes.push_back(node); @@ -43,16 +44,7 @@ public: return 0xffff; } - virtual void print(std::ostream& out, bool /* verbose */, - const std::string& /* indent */) const - { - out << "["; - for (uint32_t i=0; i<_idealNodes.size(); ++i) { - if (i != 0) out << ", "; - out << _idealNodes[i]; - } - out << "]"; - } + void print(std::ostream& out, bool, const std::string &) const override; }; /** diff --git a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp index 86c63f289e4..ca7ab742278 100644 --- a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp +++ b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp @@ -2,10 +2,28 @@ #include "idealnodecalculatorimpl.h" #include <vespa/vespalib/util/exceptions.h> +#include <ostream> namespace storage { namespace lib { +IdealNodeList::IdealNodeList() : + _idealNodes() +{ } + +IdealNodeList::~IdealNodeList() { } + +void +IdealNodeList::print(std::ostream& out, bool , const std::string &) const +{ + out << "["; + for (uint32_t i=0; i<_idealNodes.size(); ++i) { + if (i != 0) out << ", "; + out << _idealNodes[i]; + } + out << "]"; +} + IdealNodeCalculatorImpl::IdealNodeCalculatorImpl() : _distribution(0), _clusterState(0) |