diff options
-rw-r--r-- | vdslib/src/tests/state/CMakeLists.txt | 1 | ||||
-rw-r--r-- | vdslib/src/tests/state/grouptest.cpp | 430 | ||||
-rw-r--r-- | vdslib/src/vespa/vdslib/state/idealgroup.cpp | 26 | ||||
-rw-r--r-- | vdslib/src/vespa/vdslib/state/idealgroup.h | 61 |
4 files changed, 96 insertions, 422 deletions
diff --git a/vdslib/src/tests/state/CMakeLists.txt b/vdslib/src/tests/state/CMakeLists.txt index 6308e06e3f1..0e057d12226 100644 --- a/vdslib/src/tests/state/CMakeLists.txt +++ b/vdslib/src/tests/state/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(vdslib_teststate SOURCES cluster_state_bundle_test.cpp clusterstatetest.cpp + grouptest.cpp nodestatetest.cpp DEPENDS vdslib diff --git a/vdslib/src/tests/state/grouptest.cpp b/vdslib/src/tests/state/grouptest.cpp index 94694154b83..b9f2da38564 100644 --- a/vdslib/src/tests/state/grouptest.cpp +++ b/vdslib/src/tests/state/grouptest.cpp @@ -1,393 +1,153 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vdslib/state/nodetype.h> -#include <vdslib/state/group.h> -#include <vespa/vdslib/state/idealgroup.h> -#include <vespa/vespalib/util/exceptions.h> -#include <cppunit/extensions/HelperMacros.h> -#include <boost/lexical_cast.hpp> -#include <algorithm> -using namespace std; -using namespace vdslib; +#include <vespa/vdslib/distribution/group.h> +#include <vespa/vespalib/gtest/gtest.h> -class GroupTest : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE(GroupTest); - CPPUNIT_TEST(testTree); - CPPUNIT_TEST(testSetNodes); - CPPUNIT_TEST(testOperators); - CPPUNIT_TEST(testStarConversion); - CPPUNIT_TEST(testGroupIndexOrder); - CPPUNIT_TEST(testNodeIndexOrder); - CPPUNIT_TEST(testIdealGroupRedundancyOrder); - CPPUNIT_TEST(testIdealGroupScoreOrder); - CPPUNIT_TEST(testIdealGroupPickingConsistency); - CPPUNIT_TEST_SUITE_END(); +using namespace storage::lib; -public: -protected: - void testTree(); - void testSetNodes(); - void testOperators(); - void testStarConversion(); - void testGroupIndexOrder(); - void testNodeIndexOrder(); - void testIdealGroupRedundancyOrder(); - void testIdealGroupScoreOrder(); - void testIdealGroupPickingConsistency(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION( GroupTest ); - -#define MAKEGROUP(group, name, index, distribution) \ - Group group; \ - group.setName(name); \ - group.setIndex(index); \ - group.setDistribution(distribution); - -#define MAKEGROUPPTR(group, name, index, distribution) \ - Group* group = new Group(); \ - group->setName(name); \ - group->setIndex(index); \ - group->setDistribution(distribution); - -#define VERIFY(group, expected) { \ - std::ostringstream ost; \ - group.serialize(ost, true); \ - CPPUNIT_ASSERT_EQUAL(std::string(expected), ost.str()); \ -} - -void -GroupTest::testTree() +Group::UP +make_group(uint16_t index, vespalib::stringref distribution, uint16_t redundancy = 1) { - uint16_t numGroups=10; - - std::vector<Group*> groups; - for (uint16_t i=0; i< numGroups; ++i){ - std::string name = vespalib::make_string("groupname%d", i); - MAKEGROUPPTR(g, name, i, ""); - if (i >= numGroups/2) { - std::string storageIndexes = - vespalib::make_string("[%d-%d]", i, i); - g->setNodes(NodeType::STORAGE, storageIndexes); - std::string distributorIndexes = - vespalib::make_string("[%d-%d]", i-1, i-1); - g->setNodes(NodeType::DISTRIBUTOR, distributorIndexes); - } - groups.push_back(g); - } - - for (uint16_t i=numGroups-1; i>0; --i){ - uint16_t child = i; - uint16_t parent = i/2; - groups[child]->setParent(groups[parent]); - groups[parent]->addSubGroup(groups[child]); - } - - //groups[0]->print(std::cerr, true, ""); std::cerr << "\n"; - - CPPUNIT_ASSERT_EQUAL((uint16_t)(numGroups-1), - groups[0]->getMaxIndex(NodeType::STORAGE)); - CPPUNIT_ASSERT_EQUAL((uint16_t)(numGroups-2), - groups[0]->getMaxIndex(NodeType::DISTRIBUTOR)); - - for (uint16_t i=0; i<numGroups+1; ++i){ - if (i<numGroups/2 || i>=numGroups) { - CPPUNIT_ASSERT(!groups[0]->containsNode(NodeType::STORAGE, i)); - } else { - CPPUNIT_ASSERT(groups[0]->containsNode(NodeType::STORAGE, i)); - } - } - - delete groups[0]; + return std::make_unique<Group>(index, "group", Group::Distribution(distribution), redundancy); } -void -GroupTest::testSetNodes() +TEST(GroupTest, test_operators) { - MAKEGROUP(group, "group", 0, ""); - std::string noNodes(" ( name:group index:0 )"); - std::string threeNodes(" ( name:group index:0 distributor:[0-2] )"); { - try { - group.setNodes(NodeType::DISTRIBUTOR, "a"); - CPPUNIT_ASSERT(false); - } catch (boost::bad_lexical_cast& e) { - VERIFY(group, noNodes); - } - try { - group.setNodes(NodeType::DISTRIBUTOR, "[b]"); - CPPUNIT_ASSERT(false); - } catch (boost::bad_lexical_cast& e) { - VERIFY(group, noNodes); - } - try { - group.setNodes(NodeType::DISTRIBUTOR, "[0-c]"); - CPPUNIT_ASSERT(false); - } catch (boost::bad_lexical_cast& e) { - VERIFY(group, noNodes); - } - try { - group.setNodes(NodeType::DISTRIBUTOR, "[-1-6]"); - CPPUNIT_ASSERT(false); - } catch (boost::bad_lexical_cast& e) { - VERIFY(group, noNodes); - } - group.setNodes(NodeType::DISTRIBUTOR, "[5-3]"); - //group.print(std::cerr, true, ""); std::cerr << "\n"; - VERIFY(group, noNodes); - - group.setNodes(NodeType::DISTRIBUTOR, "3"); - VERIFY(group, threeNodes); - - group.setNodes(NodeType::DISTRIBUTOR, "[0-2]"); - VERIFY(group, threeNodes); - - group.setNodes(NodeType::DISTRIBUTOR, "[0,1,2]"); - VERIFY(group, threeNodes); - - group.setNodes(NodeType::DISTRIBUTOR, "[,0,1,2,]"); - VERIFY(group, threeNodes); - - group.setNodes(NodeType::DISTRIBUTOR, "[0,0-2]"); - VERIFY(group, threeNodes); + auto g0 = make_group(0, "1|*|*"); + auto g1 = make_group(0, "1|*|*"); + EXPECT_EQ(*g0, *g1); } -} - -void -GroupTest::testOperators() -{ { - MAKEGROUP(g0, "group", 0, "1|*|*"); - MAKEGROUP(g1, "group", 1, "1|*|*"); - CPPUNIT_ASSERT(g0 != g1); + auto g0 = make_group(0, "1|*|*"); + auto g1 = make_group(1, "1|*|*"); + EXPECT_TRUE(!(*g0 == *g1)); } { - MAKEGROUP(g0, "group", 0, "1|*"); - MAKEGROUP(g1, "group", 0, "1|*|*"); - CPPUNIT_ASSERT(g0 != g1); + auto g0 = make_group(0, "1|*"); + auto g1 = make_group(0, "1|*|*"); + EXPECT_TRUE(!(*g0 == *g1)); } { - MAKEGROUPPTR(g0, "group", 0, "1|*"); - MAKEGROUPPTR(g1, "group", 1, "1|*|*"); - g0->addSubGroup(g1); - MAKEGROUPPTR(g2, "group", 0, "1|*"); - CPPUNIT_ASSERT(g0 != g2); - delete g0; delete g2; + auto g0 = make_group(0, "1|*"); + auto g1 = make_group(1, "1|*|*"); + g0->addSubGroup(std::move(g1)); + auto g2 = make_group(0, "1|*"); + EXPECT_TRUE(!(*g0 == *g2)); } } - -void -GroupTest::testStarConversion() +TEST(GroupTest, test_star_conversion) { { - MAKEGROUP(g, "group", 0, "*"); - std::vector<double> distribution = g.getDistribution(3); - CPPUNIT_ASSERT_EQUAL((size_t) 1, distribution.size()); - CPPUNIT_ASSERT_EQUAL((double) 3, distribution[0]); + auto g = make_group(0, "*", 3); + const auto& distribution = g->getDistribution(3); + ASSERT_EQ(1, distribution.size()); + EXPECT_EQ(3, distribution[0]); } { - MAKEGROUP(g, "group", 0, "1|*|*"); - std::vector<double> distribution = g.getDistribution(5); - CPPUNIT_ASSERT_EQUAL((size_t) 3, distribution.size()); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[0]); - CPPUNIT_ASSERT_EQUAL((double) 2, distribution[1]); - CPPUNIT_ASSERT_EQUAL((double) 2, distribution[2]); + auto g = make_group(0, "1|*|*", 5); + const auto& distribution = g->getDistribution(5); + ASSERT_EQ(3, distribution.size()); + EXPECT_EQ(2, distribution[0]); + EXPECT_EQ(2, distribution[1]); + EXPECT_EQ(1, distribution[2]); } { - MAKEGROUP(g, "group", 0, "1|*|*"); - std::vector<double> distribution = g.getDistribution(3); - CPPUNIT_ASSERT_EQUAL((size_t) 3, distribution.size()); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[0]); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[1]); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[2]); + auto g = make_group(0, "1|*|*", 3); + const auto& distribution = g->getDistribution(3); + ASSERT_EQ(3, distribution.size()); + EXPECT_EQ(1, distribution[0]); + EXPECT_EQ(1, distribution[1]); + EXPECT_EQ(1, distribution[2]); } { - MAKEGROUP(g, "group", 0, "1|*"); - std::vector<double> distribution = g.getDistribution(3); - CPPUNIT_ASSERT_EQUAL((size_t) 2, distribution.size()); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[0]); - CPPUNIT_ASSERT_EQUAL((double) 2, distribution[1]); + auto g = make_group(0, "1|*", 3); + const auto& distribution = g->getDistribution(3); + ASSERT_EQ(2, distribution.size()); + EXPECT_EQ(2, distribution[0]); + EXPECT_EQ(1, distribution[1]); } { - MAKEGROUP(g, "group", 0, "4|*"); - std::vector<double> distribution = g.getDistribution(3); - CPPUNIT_ASSERT_EQUAL((size_t) 1, distribution.size()); - CPPUNIT_ASSERT_EQUAL((double) 3, distribution[0]); + auto g = make_group(0, "4|*", 3); + const auto& distribution = g->getDistribution(3); + ASSERT_EQ(2, distribution.size()); + EXPECT_EQ(2, distribution[0]); + EXPECT_EQ(1, distribution[1]); } { - MAKEGROUP(g, "group", 0, "2|*"); - std::vector<double> distribution = g.getDistribution(3); - CPPUNIT_ASSERT_EQUAL((size_t) 2, distribution.size()); - CPPUNIT_ASSERT_EQUAL((double) 2, distribution[0]); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[1]); + auto g = make_group(0, "2|*", 3); + const auto& distribution = g->getDistribution(3); + ASSERT_EQ(2, distribution.size()); + EXPECT_EQ(2, distribution[0]); + EXPECT_EQ(1, distribution[1]); } { - MAKEGROUP(g, "group", 0, "2|*"); - std::vector<double> distribution = g.getDistribution(0); - CPPUNIT_ASSERT_EQUAL((size_t) 0, distribution.size()); + auto g = make_group(0, "2|*", 0); + const auto& distribution = g->getDistribution(0); + ASSERT_EQ(0, distribution.size()); } { - MAKEGROUP(g, "group", 0, "*|*"); - std::vector<double> distribution = g.getDistribution(3); - CPPUNIT_ASSERT_EQUAL((size_t) 2, distribution.size()); - CPPUNIT_ASSERT_EQUAL((double) 2, distribution[0]); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[1]); + auto g = make_group(0, "*|*", 3); + const auto& distribution = g->getDistribution(3); + ASSERT_EQ(2, distribution.size()); + EXPECT_EQ(2, distribution[0]); + EXPECT_EQ(1, distribution[1]); } { - MAKEGROUP(g, "group", 0, "*|*|*"); - std::vector<double> distribution = g.getDistribution(4); - CPPUNIT_ASSERT_EQUAL((size_t) 3, distribution.size()); - CPPUNIT_ASSERT_EQUAL((double) 2, distribution[0]); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[1]); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[2]); + auto g = make_group(0, "*|*|*", 4); + const auto& distribution = g->getDistribution(4); + ASSERT_EQ(3, distribution.size()); + EXPECT_EQ(2, distribution[0]); + EXPECT_EQ(1, distribution[1]); + EXPECT_EQ(1, distribution[2]); } { - MAKEGROUP(g, "group", 0, "*|*|*"); - std::vector<double> distribution = g.getDistribution(5); - CPPUNIT_ASSERT_EQUAL((size_t) 3, distribution.size()); - CPPUNIT_ASSERT_EQUAL((double) 2, distribution[0]); - CPPUNIT_ASSERT_EQUAL((double) 2, distribution[1]); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[2]); + auto g = make_group(0, "*|*|*", 5); + const auto& distribution = g->getDistribution(5); + ASSERT_EQ(3, distribution.size()); + EXPECT_EQ(2, distribution[0]); + EXPECT_EQ(2, distribution[1]); + EXPECT_EQ(1, distribution[2]); } { - MAKEGROUP(g, "group", 0, "*|*|*"); - std::vector<double> distribution = g.getDistribution(12); // Shall be evenly divided - CPPUNIT_ASSERT_EQUAL((size_t) 3, distribution.size()); - CPPUNIT_ASSERT_EQUAL((double) 4, distribution[0]); - CPPUNIT_ASSERT_EQUAL((double) 4, distribution[1]); - CPPUNIT_ASSERT_EQUAL((double) 4, distribution[2]); + auto g = make_group(0, "*|*|*", 12); + const auto& distribution = g->getDistribution(12); // Shall be evenly divided + ASSERT_EQ(3, distribution.size()); + EXPECT_EQ(4, distribution[0]); + EXPECT_EQ(4, distribution[1]); + EXPECT_EQ(4, distribution[2]); } { - MAKEGROUP(g, "group", 0, "*|*|*|*"); - std::vector<double> distribution = g.getDistribution(5); - CPPUNIT_ASSERT_EQUAL((size_t) 4, distribution.size()); - CPPUNIT_ASSERT_EQUAL((double) 2, distribution[0]); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[1]); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[2]); - CPPUNIT_ASSERT_EQUAL((double) 1, distribution[3]); + auto g = make_group(0, "*|*|*|*", 5); + const auto& distribution = g->getDistribution(5); + ASSERT_EQ(4, distribution.size()); + EXPECT_EQ(2, distribution[0]); + EXPECT_EQ(1, distribution[1]); + EXPECT_EQ(1, distribution[2]); + EXPECT_EQ(1, distribution[3]); } - - - } -void -GroupTest::testGroupIndexOrder() +TEST(GroupTest, test_group_index_order) { uint16_t numGroups=10; - std::vector<Group*> groups; + std::vector<Group::UP> groups; for (uint16_t i=0; i< numGroups; ++i){ - MAKEGROUPPTR(g, "group", i, "5.3|*"); - groups.push_back(g); + groups.push_back(make_group(i, "5|*")); } for (uint16_t i=1; i< 10; ++i){ - groups[0]->addSubGroup(groups[numGroups-i]); + groups[0]->addSubGroup(std::move(groups[numGroups-i])); } uint16_t last=0; - for (uint16_t i=0; i< groups[0]->getNumSubGroups(); ++i){ - CPPUNIT_ASSERT(last < groups[0]->getSubGroup(i)->getIndex()); - last = groups[0]->getSubGroup(i)->getIndex(); + for (const auto& sub_group : groups[0]->getSubGroups()) { + uint16_t index = sub_group.second->getIndex(); + ASSERT_EQ(index, sub_group.first); + EXPECT_LE(last, index); + last = index; } - delete groups[0]; } - -void -GroupTest::testNodeIndexOrder() -{ - uint16_t numNodes=10; - MAKEGROUP(g, "group", 0, "5.3|*"); - - for (uint16_t i=0; i< numNodes; ++i){ - g.addNode(NodeType::DISTRIBUTOR, numNodes-i); - } - - uint16_t last=0; - const std::vector<uint16_t>& nodes = g.getNodes(NodeType::DISTRIBUTOR); - CPPUNIT_ASSERT_EQUAL((size_t)numNodes, nodes.size()); - for (uint16_t i=0; i< nodes.size(); ++i){ - CPPUNIT_ASSERT(last < nodes[i]); - last = nodes[i]; - } -} - - -void -GroupTest::testIdealGroupRedundancyOrder() -{ - MAKEGROUPPTR(g0, "group0", 0, "1|2|*"); - MAKEGROUPPTR(g1, "group1", 1, "*"); - MAKEGROUPPTR(g2, "group2", 2, "*"); - MAKEGROUPPTR(g3, "group3", 3, "*"); - - g0->addSubGroup(g1); - g0->addSubGroup(g2); - g0->addSubGroup(g3); - - std::vector<IdealGroup> idealGroups; - g0->getIdealGroups(6, 100, idealGroups); - - for (uint16_t i=0; i<idealGroups.size(); ++i){ - CPPUNIT_ASSERT_EQUAL((double) (3-i), idealGroups[i].getRedundancy().getValue()); - } - delete g0; -} - -void -GroupTest::testIdealGroupScoreOrder() -{ - MAKEGROUPPTR(g0, "group0", 0, "1|2|*"); - MAKEGROUPPTR(g1, "group1", 1, "*"); - MAKEGROUPPTR(g2, "group2", 2, "*"); - MAKEGROUPPTR(g3, "group3", 3, "*"); - - g0->addSubGroup(g1); - g0->addSubGroup(g2); - g0->addSubGroup(g3); - - std::vector<IdealGroup> idealGroups; - g0->getIdealGroups(6, 100, idealGroups); - - std::sort(idealGroups.rbegin(), idealGroups.rend(), vdslib::IdealGroup::sortScore); - double last=1.0; - for (uint16_t g=0; g< idealGroups.size(); ++g){ - CPPUNIT_ASSERT(last >= idealGroups[g].getScore()); - last = idealGroups[g].getScore(); - CPPUNIT_ASSERT_EQUAL((double) (g+1), idealGroups[g].getRedundancy().getValue()); - } - delete g0; -} - - -void -GroupTest::testIdealGroupPickingConsistency() -{ - MAKEGROUPPTR(g0, "group0", 0, "1|2|*"); - MAKEGROUPPTR(g1, "group1", 1, "*"); - MAKEGROUPPTR(g2, "group2", 2, "*"); - MAKEGROUPPTR(g3, "group3", 3, "*"); - - g0->addSubGroup(g1); - g0->addSubGroup(g2); - g0->addSubGroup(g3); - - std::vector<IdealGroup> idealGroups1; - g0->getIdealGroups(6, 100, idealGroups1); - - - std::vector<IdealGroup> idealGroups2; - g0->getIdealGroups(6, 100, idealGroups2); - - CPPUNIT_ASSERT_EQUAL((size_t)3, idealGroups1.size()); - CPPUNIT_ASSERT_EQUAL((size_t)3, idealGroups2.size()); - - for (uint16_t g=0; g< idealGroups1.size(); ++g){ - CPPUNIT_ASSERT_EQUAL(idealGroups1[g].getGroup(), - idealGroups2[g].getGroup()); - } - delete g0; -} diff --git a/vdslib/src/vespa/vdslib/state/idealgroup.cpp b/vdslib/src/vespa/vdslib/state/idealgroup.cpp deleted file mode 100644 index 9dc77834141..00000000000 --- a/vdslib/src/vespa/vdslib/state/idealgroup.cpp +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vdslib/state/idealgroup.h> - -namespace storage { -namespace lib { - -IdealGroup::IdealGroup(const Group& g, double score, double redundancy) - : _group(&g), - _score(score), - _redundancy(redundancy) -{ -} - -void -IdealGroup::print(std::ostream& out, bool verbose, - const std::string& indent) const { - - out << indent << "\nredundancy : " << _redundancy << "\n"; - out << indent << "score : " << _score << "\n"; - out << indent << "group : \n" ; - _group->print(out, verbose, indent + " "); -} - -} // lib -} // storage diff --git a/vdslib/src/vespa/vdslib/state/idealgroup.h b/vdslib/src/vespa/vdslib/state/idealgroup.h deleted file mode 100644 index 9dcfde3e185..00000000000 --- a/vdslib/src/vespa/vdslib/state/idealgroup.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class vdslib::IdealGroup - * - * Points to a Group object that has been picked for - * distribution with a given redundancy. - * - * - */ -#pragma once - -#include <vdslib/state/group.h> -#include <vespa/vespalib/objects/floatingpointtype.h> - -namespace storage { -namespace lib { - -class IdealGroup : public document::Printable -{ - const Group* _group; - double _score; - double _redundancy; - -public: - IdealGroup() {}; - - IdealGroup(const Group& group, double score, double redundancy); - - virtual void print(std::ostream& out, bool verbose, - const std::string& indent) const; - - double getRedundancy() const {return _redundancy;} - - void setRedundancy(double redundancy) {_redundancy = redundancy;} - - double getScore() const {return _score;} - - static bool sortScore (IdealGroup ig1, IdealGroup ig2) - { - return (ig1._score<ig2._score); - } - - static bool sortRedundancy (IdealGroup ig1, IdealGroup ig2) - { - return (ig1._redundancy<ig2._redundancy); - } - - const std::vector<uint16_t>& getNodes() const - { - return _group->getNodes(); - } - - const Group& getGroup() const { - return *_group; - } - -}; - -} // lib -} // storage - |