From a393a5ee43c92fb6e89af853b52d1f4a89e1d447 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Thu, 23 May 2019 10:55:54 +0000 Subject: Rewrite vdslib tests from cppunit to gtest. Note that testSkew() and testSkewWithDown() have been inactive for 9 years and are removed. testEmptyAndCopy() has never been executed and is also removed. --- vdslib/src/tests/.gitignore | 3 - vdslib/src/tests/CMakeLists.txt | 15 +- vdslib/src/tests/distribution/CMakeLists.txt | 1 + vdslib/src/tests/distribution/distributiontest.cpp | 856 +++++++-------------- vdslib/src/tests/distribution/grouptest.cpp | 63 +- .../distribution/idealnodecalculatorimpltest.cpp | 26 +- vdslib/src/tests/testrunner.cpp | 14 - 7 files changed, 292 insertions(+), 686 deletions(-) delete mode 100644 vdslib/src/tests/testrunner.cpp diff --git a/vdslib/src/tests/.gitignore b/vdslib/src/tests/.gitignore index 3a2dc959fb6..06ba0164eb2 100644 --- a/vdslib/src/tests/.gitignore +++ b/vdslib/src/tests/.gitignore @@ -1,7 +1,4 @@ .depend Makefile -datadistributiontest test.vlog -testrunner vdslib_gtest_runner_app -vdslib_testrunner_app diff --git a/vdslib/src/tests/CMakeLists.txt b/vdslib/src/tests/CMakeLists.txt index 084320ee059..f552808f97c 100644 --- a/vdslib/src/tests/CMakeLists.txt +++ b/vdslib/src/tests/CMakeLists.txt @@ -8,6 +8,7 @@ vespa_add_executable(vdslib_gtest_runner_app TEST DEPENDS vdslib_bucketdistributiontest vdslib_containertest + vdslib_testdistribution vdslib_teststate vdslib_testthread gtest @@ -17,17 +18,3 @@ vespa_add_test( NAME vdslib_gtest_runner_app COMMAND vdslib_gtest_runner_app ) - -# Runner for unit tests written in CppUnit (DEPRECATED). -vespa_add_executable(vdslib_testrunner_app TEST - SOURCES - testrunner.cpp - DEPENDS - vdslib_testdistribution -) - -# TODO: Test with a larger chunk size to parallelize test suite runs -vespa_add_test( - NAME vdslib_testrunner_app - COMMAND vdslib_testrunner_app -) diff --git a/vdslib/src/tests/distribution/CMakeLists.txt b/vdslib/src/tests/distribution/CMakeLists.txt index feb8d458203..cdcfcfce5f9 100644 --- a/vdslib/src/tests/distribution/CMakeLists.txt +++ b/vdslib/src/tests/distribution/CMakeLists.txt @@ -6,4 +6,5 @@ vespa_add_library(vdslib_testdistribution idealnodecalculatorimpltest.cpp DEPENDS vdslib + gtest ) diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index 76feec9ffbb..80f28af17b5 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -1,111 +1,32 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include +#include +#include +#include +#include #include #include #include -#include #include +#include #include -#include +#include #include #include -#include -#include -#include -#include -#include -#include #include #include #include namespace storage::lib { -struct DistributionTest : public CppUnit::TestFixture { - void testVerifyJavaDistributions(); - void testVerifyJavaDistributions2(); - void testDiskSkewLocal(); - void testDiskSkewGlobal(); - void testDiskIntersection(); - void testDown(); - void testInitializing(); - void testDiskDown(); - void testDiskDownMaintenance(); - void testDiskCapacityWeights(); - void testUnchangedDistribution(); - - void testSerializeDeserialize(); - - void testSkew(); - void testDistribution(); - void testGreedyDistribution(); - void testSkewWithDown(); - void testMove(); - void testMoveConstraints(); - void testMinimalDataMovement(); - void testDistributionBits(); - - void testRedundancyHierarchicalDistribution(); - void testHierarchicalDistribution(); - void testHierarchicalDistributionPerformance(); - void testHierarchicalNoRedistribution(); - void testGroupCapacity(); - - void testHighSplitBit(); - - void testActivePerGroup(); - void testHierarchicalDistributeLessThanRedundancy(); - - void testEmptyAndCopy(); - - CPPUNIT_TEST_SUITE(DistributionTest); - CPPUNIT_TEST(testVerifyJavaDistributions); - CPPUNIT_TEST(testVerifyJavaDistributions2); - CPPUNIT_TEST(testDiskSkewLocal); - CPPUNIT_TEST(testDiskSkewGlobal); - CPPUNIT_TEST(testDiskIntersection); - CPPUNIT_TEST(testMove); - CPPUNIT_TEST(testMoveConstraints); - CPPUNIT_TEST(testDown); - CPPUNIT_TEST(testInitializing); - CPPUNIT_TEST(testDiskDown); - CPPUNIT_TEST(testDiskDownMaintenance); - CPPUNIT_TEST(testDiskCapacityWeights); - CPPUNIT_TEST(testUnchangedDistribution); - CPPUNIT_TEST(testSerializeDeserialize); - - CPPUNIT_TEST(testRedundancyHierarchicalDistribution); - CPPUNIT_TEST(testHierarchicalDistribution); - // CPPUNIT_TEST(testHierarchicalDistributionPerformance); - CPPUNIT_TEST(testHierarchicalNoRedistribution); - CPPUNIT_TEST(testHierarchicalDistributeLessThanRedundancy); - CPPUNIT_TEST(testDistributionBits); - CPPUNIT_TEST(testGroupCapacity); - - CPPUNIT_TEST(testHighSplitBit); - CPPUNIT_TEST(testActivePerGroup); - - // Skew tests. Should probably be in separate test file. - /* - CPPUNIT_TEST(testSkew); - CPPUNIT_TEST(testDistribution); - CPPUNIT_TEST(testGreedyDistribution); - CPPUNIT_TEST(testSkewWithDown); - */ - - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(DistributionTest); - template T readConfig(const config::ConfigUri & uri) { return *config::ConfigGetter::getConfig(uri.getConfigId(), uri.getContext()); } -void -DistributionTest::testVerifyJavaDistributions() +TEST(DistributionTest, test_verify_java_distributions) { std::vector tests; tests.push_back("capacity"); @@ -116,15 +37,15 @@ DistributionTest::testVerifyJavaDistributions() std::string test = tests[i]; ClusterState state; { - std::ifstream in(TEST_PATH("distribution/testdata/java_" + test + ".state").c_str()); + std::ifstream in("distribution/testdata/java_" + test + ".state"); std::string mystate; in >> mystate; in.close(); state = ClusterState(mystate); } Distribution distr(readConfig( - "file:" + TEST_PATH("distribution/testdata/java_") + test + ".cfg")); - std::ofstream of((TEST_PATH("distribution/testdata/cpp_") + test + ".distribution").c_str()); + "file:distribution/testdata/java_" + test + ".cfg")); + std::ofstream of("distribution/testdata/cpp_" + test + ".distribution"); long maxBucket = 1; long mask = 0; @@ -155,72 +76,67 @@ DistributionTest::testVerifyJavaDistributions() } of.close(); std::ostringstream cmd; - cmd << "diff -u " << TEST_PATH("distribution/testdata/cpp_") << test << ".distribution " - << TEST_PATH("distribution/testdata/java_") << test << ".distribution"; + cmd << "diff -u " << "distribution/testdata/cpp_" << test << ".distribution " + << "distribution/testdata/java_" << test << ".distribution"; int result = system(cmd.str().c_str()); - CPPUNIT_ASSERT_EQUAL_MSG("Failed distribution sync test: " + test, - 0, result); + EXPECT_EQ(0, result) << "Failed distribution sync test: " + test; } } namespace { - struct ExpectedResult { - ExpectedResult() { } - ExpectedResult(const ExpectedResult &) = default; - ExpectedResult & operator = (const ExpectedResult &) = default; - ExpectedResult(ExpectedResult &&) = default; - ExpectedResult & operator = (ExpectedResult &&) = default; - ~ExpectedResult() { } - document::BucketId bucket; - IdealNodeList nodes; - vespalib::string failure; - }; - void verifyJavaDistribution( - const vespalib::string& name, - const ClusterState& state, - const Distribution& distribution, - const NodeType& nodeType, - uint16_t redundancy, - uint16_t nodeCount, - vespalib::stringref upStates, - const std::vector results) - { - (void) nodeCount; - for (uint32_t i=0, n=results.size(); i nvect; - distribution.getIdealNodes(nodeType, state, results[i].bucket, - nvect, upStates.data(), redundancy); - IdealNodeList nodes; - for (uint32_t j=0, m=nvect.size(); j 0) { - CPPUNIT_ASSERT_EQUAL_MSG(testId, vespalib::string("NONE"), - results[i].failure); - } else { - CPPUNIT_ASSERT_EQUAL_MSG(testId, - vespalib::string("NO_DISTRIBUTORS_AVAILABLE"), - results[i].failure); - } - } catch (vespalib::Exception& e) { - CPPUNIT_ASSERT_EQUAL_MSG(testId, results[i].failure, - e.getMessage()); + +struct ExpectedResult { + ExpectedResult() { } + ExpectedResult(const ExpectedResult &) = default; + ExpectedResult & operator = (const ExpectedResult &) = default; + ExpectedResult(ExpectedResult &&) = default; + ExpectedResult & operator = (ExpectedResult &&) = default; + ~ExpectedResult() { } + document::BucketId bucket; + IdealNodeList nodes; + vespalib::string failure; +}; + +void +verifyJavaDistribution(const vespalib::string& name, + const ClusterState& state, + const Distribution& distribution, + const NodeType& nodeType, + uint16_t redundancy, + uint16_t nodeCount, + vespalib::stringref upStates, + const std::vector results) +{ + (void) nodeCount; + for (uint32_t i=0, n=results.size(); i nvect; + distribution.getIdealNodes(nodeType, state, results[i].bucket, + nvect, upStates.data(), redundancy); + IdealNodeList nodes; + for (uint32_t j=0, m=nvect.size(); j 0) { + EXPECT_EQ(vespalib::string("NONE"), results[i].failure) << testId; + } else { + EXPECT_EQ(vespalib::string("NO_DISTRIBUTORS_AVAILABLE"), results[i].failure) << testId; } + } catch (vespalib::Exception& e) { + EXPECT_EQ(results[i].failure, e.getMessage()) << testId; } } -} // anonymous +} + +} auto readFile(const std::string & filename) { vespalib::File file(filename); @@ -229,15 +145,14 @@ auto readFile(const std::string & filename) { std::vector buf(file.getFileSize()); off_t read = file.read(&buf[0], buf.size(), 0); - CPPUNIT_ASSERT_EQUAL(read, file.getFileSize()); + assert(read == file.getFileSize()); return buf; } -void -DistributionTest::testVerifyJavaDistributions2() +TEST(DistributionTest, test_verify_java_distributions_2) { vespalib::DirectoryList files( - vespalib::listDirectory(TEST_PATH("distribution/testdata"))); + vespalib::listDirectory("distribution/testdata")); for (uint32_t i=0, n=files.size(); i _distribution; - uint32_t _bucketsToTest; - const char* _upStates; - uint16_t _redundancy; - - Test(); - ~Test(); - - Test& state(const std::string& s) { - _state = s; - return *this; - } - Test& upStates(const char* ups) { - _upStates = ups; - return *this; - } +struct MyTest { + const NodeType* _nodeType; + std::string _state; + std::unique_ptr _distribution; + uint32_t _bucketsToTest; + const char* _upStates; + uint16_t _redundancy; - Test& nodeType(const NodeType& type) { - _nodeType = &type; - return *this; - } + MyTest(); + ~MyTest(); - Test& distribution(Distribution* d) { - _distribution.reset(d); - return *this; - } + MyTest& state(const std::string& s) { + _state = s; + return *this; + } - std::vector getNodeCounts() const { - std::vector result(10, 0); - for (uint32_t i=0; i<_bucketsToTest; ++i) { - document::BucketId bucket(16, i); - std::vector nodes; - ClusterState clusterState(_state); - _distribution->getIdealNodes( - *_nodeType, clusterState, bucket, nodes, - _upStates, _redundancy); - for (uint32_t j=0; j getNodeCounts() const { + std::vector result(10, 0); + for (uint32_t i=0; i<_bucketsToTest; ++i) { + document::BucketId bucket(16, i); + std::vector nodes; + ClusterState clusterState(_state); + _distribution->getIdealNodes( + *_nodeType, clusterState, bucket, nodes, + _upStates, _redundancy); + for (uint32_t j=0; j getDiskCounts(uint16_t node) const { - std::vector result(3, 0); - for (uint32_t i=0; i<_bucketsToTest; ++i) { - document::BucketId bucket(16, i); - std::vector nodes; - ClusterState clusterState(_state); - _distribution->getIdealNodes( - *_nodeType, clusterState, bucket, nodes, - _upStates, _redundancy); - for (uint32_t j=0; jgetIdealDisk( - nodeState, node, bucket, - Distribution::IDEAL_DISK_EVEN_IF_DOWN); - ++result[disk]; - } + return result; + } + std::vector getDiskCounts(uint16_t node) const { + std::vector result(3, 0); + for (uint32_t i=0; i<_bucketsToTest; ++i) { + document::BucketId bucket(16, i); + std::vector nodes; + ClusterState clusterState(_state); + _distribution->getIdealNodes( + *_nodeType, clusterState, bucket, nodes, + _upStates, _redundancy); + for (uint32_t j=0; jgetIdealDisk( + nodeState, node, bucket, + Distribution::IDEAL_DISK_EVEN_IF_DOWN); + ++result[disk]; } } - return result; } - }; - - Test::Test() - : _nodeType(&NodeType::STORAGE), - _state("distributor:10 storage:10"), - _distribution(new Distribution(Distribution::getDefaultDistributionConfig(3, 10))), - _bucketsToTest(100), - _upStates("uir"), - _redundancy(2) - { } - Test::~Test() { } - - std::vector createNodeCountList(const std::string& source, - std::vector& vals) { - std::vector result(vals.size(), 0); - vespalib::StringTokenizer st(source, " "); - for (uint32_t i=0; i(st2[0])); - uint16_t value = vals[node]; - if (st2[1] == std::string("*")) { + return result; + } +}; + +MyTest::MyTest() + : _nodeType(&NodeType::STORAGE), + _state("distributor:10 storage:10"), + _distribution(new Distribution(Distribution::getDefaultDistributionConfig(3, 10))), + _bucketsToTest(100), + _upStates("uir"), + _redundancy(2) +{ } +MyTest::~MyTest() = default; + +std::vector createNodeCountList(const std::string& source, + std::vector& vals) { + std::vector result(vals.size(), 0); + vespalib::StringTokenizer st(source, " "); + for (uint32_t i=0; i(st2[0])); + uint16_t value = vals[node]; + if (st2[1] == std::string("*")) { + value = vals[node]; + } else if (st2[1] == std::string("+")) { + if (vals[node] > 0) { value = vals[node]; - } else if (st2[1] == std::string("+")) { - if (vals[node] > 0) { - value = vals[node]; - } else { - value = 0xffff; - } } else { - value = vespalib::lexical_cast(st2[1]); + value = 0xffff; } - result[node] = value; + } else { + value = vespalib::lexical_cast(st2[1]); } - return result; + result[node] = value; } + return result; +} + } #define ASSERT_BUCKET_NODE_COUNTS(test, result) \ @@ -428,69 +344,63 @@ namespace { std::vector cnt123(test.getNodeCounts()); \ std::vector exp123(createNodeCountList(result, cnt123)); \ /*std::cerr << "Expected " << exp123 << " Got " << cnt123 << "\n";*/ \ - CPPUNIT_ASSERT_EQUAL(exp123, cnt123); \ + EXPECT_EQ(exp123, cnt123); \ } #define ASSERT_BUCKET_DISK_COUNTS(node, test, result) \ { \ std::vector cnt123(test.getDiskCounts(node)); \ std::vector exp123(createNodeCountList(result, cnt123)); \ - CPPUNIT_ASSERT_EQUAL(exp123, cnt123); \ + EXPECT_EQ(exp123, cnt123); \ } -void -DistributionTest::testDown() +TEST(DistributionTest, test_down) { ASSERT_BUCKET_NODE_COUNTS( - Test().state("storage:10 .4.s:m .5.s:m .6.s:d .7.s:d .9.s:r") - .upStates("u"), + MyTest().state("storage:10 .4.s:m .5.s:m .6.s:d .7.s:d .9.s:r") + .upStates("u"), "0:+ 1:+ 2:+ 3:+ 8:+"); ASSERT_BUCKET_NODE_COUNTS( - Test().state("storage:10 .4.s:m .5.s:m .6.s:d .7.s:d .9.s:r") - .upStates("ur"), + MyTest().state("storage:10 .4.s:m .5.s:m .6.s:d .7.s:d .9.s:r") + .upStates("ur"), "0:+ 1:+ 2:+ 3:+ 8:+ 9:+"); } -void -DistributionTest::testDiskDown() +TEST(DistributionTest, testDiskDown) { ASSERT_BUCKET_DISK_COUNTS( 2, - Test().state("storage:10 .2.d:3 .2.d.0:d"), + MyTest().state("storage:10 .2.d:3 .2.d.0:d"), "1:+ 2:+"); } -void -DistributionTest::testSerializeDeserialize() +TEST(DistributionTest, test_serialize_deserialize) { - Test t1; - Test t2; + MyTest t1; + MyTest t2; t2.distribution(new Distribution(t1._distribution->serialize())); - CPPUNIT_ASSERT_EQUAL(t1.getNodeCounts(), t2.getNodeCounts()); + EXPECT_EQ(t1.getNodeCounts(), t2.getNodeCounts()); } -void -DistributionTest::testDiskDownMaintenance() +TEST(DistributionTest, test_disk_down_maintenance) { ASSERT_BUCKET_DISK_COUNTS( 2, - Test().state("storage:10 .2.s:m .2.d:3 .2.d.0:d").upStates("um"), + MyTest().state("storage:10 .2.s:m .2.d:3 .2.d.0:d").upStates("um"), "1:+ 2:+"); } -void -DistributionTest::testInitializing() +TEST(DistributionTest, test_initializing) { ASSERT_BUCKET_NODE_COUNTS( - Test().state("distributor:3 .0.s:i .1.s:i .2.s:i") - .upStates("ui") - .nodeType(NodeType::DISTRIBUTOR), + MyTest().state("distributor:3 .0.s:i .1.s:i .2.s:i") + .upStates("ui") + .nodeType(NodeType::DISTRIBUTOR), "0:+ 1:+ 2:+"); } -void -DistributionTest::testHighSplitBit() +TEST(DistributionTest, testHighSplitBit) { // Only 3 nodes of 10 are up => all copies should end on the 3 nodes and // none on the down nodes @@ -532,13 +442,10 @@ DistributionTest::testHighSplitBit() ost2 << "\n"; } - CPPUNIT_ASSERT_EQUAL(ost1.str(), ost2.str()); + EXPECT_EQ(ost1.str(), ost2.str()); } - - -void -DistributionTest::testDiskCapacityWeights() +TEST(DistributionTest, test_disk_capacity_weights) { uint16_t num_disks = 10; std::vector capacities(num_disks); @@ -578,13 +485,11 @@ DistributionTest::testDiskCapacityWeights() double skew = (diskDist[num_disks-1]-avg)/(diskDist[num_disks-1]); - CPPUNIT_ASSERT(skew < 0.3); + EXPECT_LT(skew, 0.3); } } - -void -DistributionTest::testDiskSkewLocal() +TEST(DistributionTest, test_disk_skew_local) { Distribution distr(Distribution::getDefaultDistributionConfig(2, 3, Distribution::MODULO_INDEX)); std::vector diskDist(100); @@ -598,12 +503,10 @@ DistributionTest::testDiskSkewLocal() std::sort(diskDist.begin(), diskDist.end()); - CPPUNIT_ASSERT((diskDist[99]-diskDist[0])/(diskDist[99]) < 0.05); - + EXPECT_LT((diskDist[99]-diskDist[0])/(diskDist[99]), 0.05); } -void -DistributionTest::testDiskSkewGlobal() +TEST(DistributionTest, test_disk_skew_global) { uint16_t num_disks = 10; uint16_t num_nodes = 10; @@ -630,13 +533,10 @@ DistributionTest::testDiskSkewGlobal() double skew = (diskDist2[num_nodes*num_disks-1]-diskDist2[0])/(diskDist2[num_nodes*num_disks-1]); - CPPUNIT_ASSERT(skew < 0.2); - + EXPECT_LT(skew, 0.2); } - -void -DistributionTest::testDiskIntersection() +TEST(DistributionTest, test_disk_intersection) { uint16_t num_disks = 8; uint16_t num_nodes = 20; @@ -667,117 +567,11 @@ DistributionTest::testDiskIntersection() if (max / 1000 > 0.5) { std::ostringstream ost; ost << "Value of " << max << " / " << 1000 << " is more than 0.5"; - CPPUNIT_FAIL(ost.str()); + FAIL() << ost.str(); } } - -void -DistributionTest::testSkew() -{ - const int buckets = 200000; - const int nodes = 50; - const size_t copies = 3; - - ClusterState systemState("storage:50"); - - Distribution distr(Distribution::getDefaultDistributionConfig(copies, nodes)); - - std::vector > > _distribution(buckets); - std::vector _nodeCount(nodes, 0); - - for (int i = 0; i < buckets; i++) { - _distribution[i].first = i * 100; - _distribution[i].second = distr.getIdealStorageNodes( - systemState, document::BucketId(26, i * 100)); - CPPUNIT_ASSERT_EQUAL(copies, _distribution[i].second.size()); - sort(_distribution[i].second.begin(), _distribution[i].second.end()); - unique(_distribution[i].second.begin(), _distribution[i].second.end()); - CPPUNIT_ASSERT_EQUAL(copies, _distribution[i].second.size()); - - for (unsigned j = 0; j < _distribution[i].second.size(); j++) { - _nodeCount[_distribution[i].second[j]]++; - } - } - -/** - for (int i = 0; i < nodes; i++) { - fprintf(stderr, "%d ", _nodeCount[i]); - } - -*/ - sort(_nodeCount.begin(), _nodeCount.end()); - -/** - // Check distribution - for (int i = 0; i < nodes; i++) { - fprintf(stderr, "%d ", _nodeCount[i]); - } -*/ - - double skew = _nodeCount[nodes - 1] - _nodeCount[0]; - skew /= _nodeCount[nodes - 1]; - // fprintf(stderr, " skew = %f\n", skew); - CPPUNIT_ASSERT_MESSAGE("Distribution skew too big (> 6%)", skew < 0.06); - -} - -// Get node with distribution farest from average -int -getMaxAbs(const std::vector distribution, double avg, int start) -{ - int max = start; - for (uint32_t i = 0; i < distribution.size(); i++) { - if (std::fabs(distribution[i]-avg) > std::fabs(distribution[max]-avg)) - max = i; - } - return max; -} - -void -getSkew(int n, std::vector distribution, int &min, int &max, double &skew) -{ - min = 0; - max = 0; - - for(int i=0; i distribution[max]) - max = i; - } - - skew = distribution[max] - distribution[min]; - skew /= distribution[max]; -} - - -double -get_K_lowest(std::vector v, int k) -{ - double lowest[k]; - - for (int i = 0; i < k; i++){ - lowest[i] = v[i]; - } - - for (uint32_t i = 0; i < v.size(); i++){ - for (int j = 0; j < k; j++){ - if( v[i] < lowest[j]){ - for (int l = k-1; l > j; l--){ - lowest[l] = lowest[l-1]; - } - lowest[j] = v[i]; - break; - } - } - } - - return lowest[k-1]; -} - -void -DistributionTest::testDistribution() +TEST(DistributionTest, test_distribution) { const int min_buckets = 1024*64; const int max_buckets = 1024*64; @@ -829,62 +623,12 @@ DistributionTest::testDistribution() double skew = _nodeCount[max] - _nodeCount[min]; skew /= _nodeCount[max]; fprintf(stderr, "%d \t skew = %f\n", n, skew); - CPPUNIT_ASSERT_MESSAGE("Distribution skew too big (> 10%)", skew < 0.1); + EXPECT_LT(skew, 0.1) << "Distribution skew too big (> 10%)"; } } } -void -DistributionTest::testSkewWithDown() -{ - const int buckets = 200000; - const int nodes = 50; - const size_t copies = 3; - - ClusterState systemState("storage:50 .5.s:d .10.s:d .15.s:d .20.s:d " - ".25.s:d .30.s:d .35.s:d .40.s:d .45.s:d"); - - Distribution distr(Distribution::getDefaultDistributionConfig(3, 50)); - - std::vector > > _distribution(buckets); - std::vector _nodeCount(nodes, 0); - - for (int i = 0; i < buckets; i++) { - _distribution[i].second.reserve(copies); - } - - for (int i = 0; i < buckets; i++) { - _distribution[i].first = i * 100; - _distribution[i].second = distr.getIdealStorageNodes( - systemState, document::BucketId(26, _distribution[i].first)); - CPPUNIT_ASSERT_EQUAL(copies, _distribution[i].second.size()); - sort(_distribution[i].second.begin(), _distribution[i].second.end()); - unique(_distribution[i].second.begin(), _distribution[i].second.end()); - CPPUNIT_ASSERT_EQUAL(copies, _distribution[i].second.size()); - - for (unsigned j = 0; j < _distribution[i].second.size(); j++) { - _nodeCount[_distribution[i].second[j]]++; - } - } - /* - // Check distribution - for (int i = 0; i < nodes; i++) { - fprintf(stderr, "%d ", _nodeCount[i]); - } - */ - sort(_nodeCount.begin(), _nodeCount.end()); - int firstUp = 0; - while (_nodeCount[firstUp] == 0) { - firstUp++; - } - double skew = _nodeCount[nodes - 1] - _nodeCount[firstUp]; - skew /= _nodeCount[nodes - 1]; - //fprintf(stderr, " skew = %f\n", skew); - CPPUNIT_ASSERT_MESSAGE("Distribution skew too big (> 5%)", skew < 0.05); -} - -void -DistributionTest::testMove() +TEST(DistributionTest, test_move) { // This test is quite fragile, it will break if the ideal state algorithm is // changed in such a way that Bucket 0x8b4f67ae remains on node 0 and 1 if @@ -897,7 +641,7 @@ DistributionTest::testMove() Distribution distr(Distribution::getDefaultDistributionConfig(2, 3)); res = distr.getIdealStorageNodes(systemState, bucket); - CPPUNIT_ASSERT_EQUAL(size_t(2), res.size()); + EXPECT_EQ(size_t(2), res.size()); } std::vector res2; @@ -909,7 +653,7 @@ DistributionTest::testMove() document::BucketId bucket(16, 0x8b4f67ae); res2 = distr.getIdealStorageNodes(systemState, bucket); - CPPUNIT_ASSERT_EQUAL(size_t(2), res2.size()); + EXPECT_EQ(size_t(2), res2.size()); } sort(res.begin(), res.end()); @@ -919,12 +663,10 @@ DistributionTest::testMove() std::vector::iterator it; it=set_difference(res.begin(), res.end(), res2.begin(), res2.end(), diff.begin()); - CPPUNIT_ASSERT_EQUAL(1, int(it-diff.begin())); - + EXPECT_EQ(1, int(it-diff.begin())); } -void -DistributionTest::testMoveConstraints() +TEST(DistributionTest, test_move_constraints) { ClusterState systemState("storage:10"); @@ -956,7 +698,7 @@ DistributionTest::testMoveConstraints() std::cerr << std::endl; } - CPPUNIT_ASSERT(initBuckets[i] == addedDownBuckets[i]); + EXPECT_EQ(initBuckets[i], addedDownBuckets[i]); } } @@ -985,8 +727,8 @@ DistributionTest::testMoveConstraints() std::cerr << std::endl; } - CPPUNIT_ASSERT_EQUAL((size_t)1, movedAway.size()); - CPPUNIT_ASSERT_EQUAL((uint16_t)0u, movedAway[0]); + ASSERT_EQ((size_t)1, movedAway.size()); + EXPECT_EQ((uint16_t)0u, movedAway[0]); } } } @@ -1008,15 +750,14 @@ DistributionTest::testMoveConstraints() initBuckets[i].begin(), initBuckets[i].end(), std::inserter(movedInto, movedInto.begin())); if (movedInto.size() > 0) { - CPPUNIT_ASSERT_EQUAL((size_t)1, movedInto.size()); - CPPUNIT_ASSERT_EQUAL((uint16_t)10, movedInto[0]); + ASSERT_EQ((size_t)1, movedInto.size()); + EXPECT_EQ((uint16_t)10, movedInto[0]); } } } } -void -DistributionTest::testDistributionBits() +TEST(DistributionTest, test_distribution_bits) { ClusterState state1("bits:16 distributor:10"); ClusterState state2("bits:19 distributor:10"); @@ -1036,11 +777,10 @@ DistributionTest::testDistributionBits() ost2 << index << " "; } - CPPUNIT_ASSERT(ost1.str() != ost2.str()); + EXPECT_NE(ost1.str(), ost2.str()); } -void -DistributionTest::testRedundancyHierarchicalDistribution() +TEST(DistributionTest, test_redundancy_hierarchical_distribution) { ClusterState state("storage:10 distributor:10"); @@ -1052,66 +792,11 @@ DistributionTest::testRedundancyHierarchicalDistribution() state, document::BucketId(16, i), "u"); uint16_t d2 = distr2.getIdealDistributorNode( state, document::BucketId(16, i), "u"); - CPPUNIT_ASSERT_EQUAL(d1, d2); - } -} -/* -void -DistributionTest::testHierarchicalDistributionPerformance() -{ - std::ostringstream ost; - ost << "redundancy 2\n" - "group[62]\n" - "group[0].name mycluster\n" - "group[0].index 0\n" - "group[0].partitions *\n" - "group[0].nodes[0]\n"; - - for (uint32_t i = 0; i < 21; ++i) { - int idx = (i * 3) + 1; - - ost << "group[" << idx << "].name rack" << i << "\n" - << "group[" << idx << "].index 0." << i << "\n" - << "group[" << idx << "].partitions 1|*\n" - << "group[" << idx << "].nodes[0]\n"; - - for (uint32_t j = 0; j < 2; ++j) { - idx++; - - ost << "group[" << idx << "].name switch" << idx << "\n" - << "group[" << idx << "].index 0." << i << "." << j << "\n" - << "group[" << idx << "].partitions 1|*\n" - << "group[" << idx << "].nodes[50]\n"; - - for (uint32_t n = 0; n < 50; ++n) { - int nIdx = (i * 100 + j * 50 + n); - ost << "group[" << idx << "].nodes[" << n << "].index " << nIdx << "\n"; - } - } - } - - std::cerr << ost.str() << "\n"; - - Distribution distr(vespa::config::content::StorDistributionConfig( - vespalib::StringTokenizer(ost.str(), "\n", "").getTokens())); - ClusterState state("distributor:2100 storage:2100"); - - uint32_t timeNow = time(NULL); - uint32_t numBuckets = 1000000; - - std::vector nodes; - for (uint32_t i = 0; i < numBuckets; i++) { - nodes = distr.getIdealStorageNodes( - state, document::BucketId(16, i), "u"); + EXPECT_EQ(d1, d2); } - - uint32_t timeSpent = time(NULL) - timeNow; - std::cerr << "Did " << numBuckets << " in " << timeSpent << " seconds. (" << ((double)numBuckets / (double)timeSpent) << " ops/sec)\n"; } -*/ -void -DistributionTest::testHierarchicalDistribution() +TEST(DistributionTest, test_hierarchical_distribution) { std::string distConfig( "redundancy 4\n" @@ -1136,12 +821,12 @@ DistributionTest::testHierarchicalDistribution() ClusterState state("distributor:6 storage:6"); for (uint32_t i = 0; i < 3; ++i) { - CPPUNIT_ASSERT_EQUAL( + EXPECT_EQ( vespalib::string("rack0"), distr.getNodeGraph().getGroupForNode(i)->getName()); } for (uint32_t i = 3; i < 6; ++i) { - CPPUNIT_ASSERT_EQUAL( + EXPECT_EQ( vespalib::string("rack1"), distr.getNodeGraph().getGroupForNode(i)->getName()); } @@ -1150,8 +835,8 @@ DistributionTest::testHierarchicalDistribution() for (uint32_t i=0; i<100; ++i) { std::vector nodes = distr.getIdealStorageNodes( state, document::BucketId(16, i), "u"); - CPPUNIT_ASSERT_EQUAL((size_t) 4, nodes.size()); - CPPUNIT_ASSERT(nodes[0] < mainNode.size()); + ASSERT_EQ((size_t) 4, nodes.size()); + EXPECT_LT(nodes[0], mainNode.size()); ++mainNode[nodes[0]]; } std::vector expectedMains(6); @@ -1161,11 +846,10 @@ DistributionTest::testHierarchicalDistribution() expectedMains[3] = 16; expectedMains[4] = 16; expectedMains[5] = 20; - CPPUNIT_ASSERT_EQUAL(expectedMains, mainNode); + EXPECT_EQ(expectedMains, mainNode); } -void -DistributionTest::testGroupCapacity() +TEST(DistributionTest, test_group_capacity) { std::string distConfig( "redundancy 1\n" @@ -1206,12 +890,11 @@ DistributionTest::testGroupCapacity() //std::cerr << "Group 0 is " << group0count << " 1 is " << group1count << "\n"; - CPPUNIT_ASSERT(group0count > 180 && group0count < 220); - CPPUNIT_ASSERT_EQUAL(1000 - group0count, group1count); + EXPECT_TRUE(group0count > 180 && group0count < 220); + EXPECT_EQ(1000 - group0count, group1count); } -void -DistributionTest::testHierarchicalNoRedistribution() +TEST(DistributionTest, test_hierarchical_no_redistribution) { std::string distConfig( "redundancy 2\n" @@ -1265,19 +948,19 @@ DistributionTest::testHierarchicalNoRedistribution() std::vector v(1000); it=set_intersection (distr[0].begin(), distr[0].end(), distr[1].begin(), distr[1].end(), v.begin()); - CPPUNIT_ASSERT_EQUAL(0, int(it-v.begin())); + EXPECT_EQ(0, int(it-v.begin())); v.clear(); it=set_intersection (distr[2].begin(), distr[2].end(), distr[3].begin(), distr[3].end(), v.begin()); - CPPUNIT_ASSERT_EQUAL(0, int(it-v.begin())); + EXPECT_EQ(0, int(it-v.begin())); v.clear(); it=set_union (distr[0].begin(), distr[0].end(), distr[1].begin(), distr[1].end(), v.begin()); - CPPUNIT_ASSERT_EQUAL(numBuckets, int(it-v.begin())); + EXPECT_EQ(numBuckets, int(it-v.begin())); v.clear(); it=set_union (distr[2].begin(), distr[2].end(), distr[3].begin(), distr[3].end(), v.begin()); - CPPUNIT_ASSERT_EQUAL(numBuckets, int(it-v.begin())); + EXPECT_EQ(numBuckets, int(it-v.begin())); v.clear(); state.setNodeState(Node(NodeType::STORAGE, 0), @@ -1289,32 +972,32 @@ DistributionTest::testHierarchicalNoRedistribution() nodes = distribution.getIdealStorageNodes( state, document::BucketId(16, i), "u"); for (uint16_t j=0; j actual(distr.splitNodesIntoLeafGroups(global)); std::vector expected; - CPPUNIT_ASSERT_EQUAL(expected, actual); + EXPECT_EQ(expected, actual); } // Random nodes { @@ -1430,12 +1113,11 @@ DistributionTest::testActivePerGroup() expected[0].push_back(1); expected[1].push_back(5); expected[1].push_back(3); - CPPUNIT_ASSERT_EQUAL(expected, actual); + EXPECT_EQ(expected, actual); } } -void -DistributionTest::testHierarchicalDistributeLessThanRedundancy() +TEST(DistributionTest, test_hierarchical_distribute_less_than_redundancy) { Distribution distr("redundancy 4\nactive_per_leaf_group true\n" + groupConfig); ClusterState state("storage:6"); @@ -1444,39 +1126,23 @@ DistributionTest::testHierarchicalDistributeLessThanRedundancy() { distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, 0), actual, "uim", 4); std::vector expected({3, 5, 1, 2}); - CPPUNIT_ASSERT_EQUAL(expected, actual); + EXPECT_EQ(expected, actual); } { distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, 0), actual, "uim", 3); std::vector expected({3, 5, 1}); - CPPUNIT_ASSERT_EQUAL(expected, actual); + EXPECT_EQ(expected, actual); } { distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, 0), actual, "uim", 2); std::vector expected({3, 1}); - CPPUNIT_ASSERT_EQUAL(expected, actual); + EXPECT_EQ(expected, actual); } { distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, 0), actual, "uim", 1); std::vector expected({3}); - CPPUNIT_ASSERT_EQUAL(expected, actual); + EXPECT_EQ(expected, actual); } } -void -DistributionTest::testEmptyAndCopy() -{ - Distribution d; - CPPUNIT_ASSERT(d.getNodeGraph().isLeafGroup()); - CPPUNIT_ASSERT_EQUAL(uint16_t(0), d.getRedundancy()); - CPPUNIT_ASSERT_EQUAL(uint16_t(0), d.getReadyCopies()); - Distribution d2(d.serialize()); - CPPUNIT_ASSERT_EQUAL(uint16_t(0), d2.getRedundancy()); - CPPUNIT_ASSERT_EQUAL(uint16_t(0), d2.getReadyCopies()); - Distribution d3(Distribution::getDefaultDistributionConfig()); - d = d3; - CPPUNIT_ASSERT_EQUAL(uint16_t(2), d.getRedundancy()); - CPPUNIT_ASSERT_EQUAL(uint16_t(1), d.getReadyCopies()); -} - } diff --git a/vdslib/src/tests/distribution/grouptest.cpp b/vdslib/src/tests/distribution/grouptest.cpp index 71d8b8bac0c..130efe15432 100644 --- a/vdslib/src/tests/distribution/grouptest.cpp +++ b/vdslib/src/tests/distribution/grouptest.cpp @@ -1,44 +1,30 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include +#include #include -#include -namespace storage { -namespace lib { - -struct GroupTest : public CppUnit::TestFixture { - void testConfigHash(); - void configHashUsesOriginalInputOrdering(); - void configHashSubgroupsAreOrderedByGroupIndex(); - - CPPUNIT_TEST_SUITE(GroupTest); - CPPUNIT_TEST(testConfigHash); - CPPUNIT_TEST(configHashUsesOriginalInputOrdering); - CPPUNIT_TEST(configHashSubgroupsAreOrderedByGroupIndex); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(GroupTest); +namespace storage::lib { namespace { - Group::UP createLeafGroup(uint16_t index, const std::string& name, - double capacity, const std::string& nodelist) - { - Group::UP group(new Group(index, name)); - group->setCapacity(capacity); - vespalib::StringTokenizer st(nodelist, ","); - std::vector nodes(st.size()); - for (uint32_t i=0; isetNodes(nodes); - return group; + +Group::UP createLeafGroup(uint16_t index, const std::string& name, + double capacity, const std::string& nodelist) +{ + Group::UP group(new Group(index, name)); + group->setCapacity(capacity); + vespalib::StringTokenizer st(nodelist, ","); + std::vector nodes(st.size()); + for (uint32_t i=0; isetNodes(nodes); + return group; +} + } -void -GroupTest::testConfigHash() +TEST(GroupTest, test_config_hash) { Group rootGroup(12, "foo", Group::Distribution("1|*"), 3); rootGroup.addSubGroup(createLeafGroup(4, "bar", 1.5, "1,4,6,8")); @@ -46,7 +32,7 @@ GroupTest::testConfigHash() rootGroup.addSubGroup(createLeafGroup(15, "ing", 1.0, "13,15")); vespalib::string expected = "(12d1|*(4c1.5;1;4;6;8)(6c1.2;3;10;11)(15;13;15))"; - CPPUNIT_ASSERT_EQUAL(expected, rootGroup.getDistributionConfigHash()); + EXPECT_EQ(expected, rootGroup.getDistributionConfigHash()); } /** @@ -54,15 +40,14 @@ GroupTest::testConfigHash() * output with the same node order as the groups were configured with, even * if their internal node list has a well-defined ordering. */ -void -GroupTest::configHashUsesOriginalInputOrdering() +TEST(GroupTest, config_hash_uses_original_input_ordering) { Group rootGroup(1, "root", Group::Distribution("1|*"), 2); rootGroup.addSubGroup(createLeafGroup(2, "fluffy", 1.0, "5,2,7,6")); rootGroup.addSubGroup(createLeafGroup(3, "bunny", 1.0, "15,10,12,11")); vespalib::string expected = "(1d1|*(2;5;2;7;6)(3;15;10;12;11))"; - CPPUNIT_ASSERT_EQUAL(expected, rootGroup.getDistributionConfigHash()); + EXPECT_EQ(expected, rootGroup.getDistributionConfigHash()); } /** @@ -71,8 +56,7 @@ GroupTest::configHashUsesOriginalInputOrdering() * * Who said anything about internal consistency, anyway? */ -void -GroupTest::configHashSubgroupsAreOrderedByGroupIndex() +TEST(GroupTest, config_hash_subgroups_are_ordered_by_group_index) { Group rootGroup(1, "root", Group::Distribution("1|*"), 2); rootGroup.addSubGroup(createLeafGroup(5, "fluffy", 1.0, "5,2,7,6")); @@ -80,8 +64,7 @@ GroupTest::configHashSubgroupsAreOrderedByGroupIndex() rootGroup.addSubGroup(createLeafGroup(4, "kitten", 1.0, "3,4,8")); vespalib::string expected = "(1d1|*(3;15;10;12;11)(4;3;4;8)(5;5;2;7;6))"; - CPPUNIT_ASSERT_EQUAL(expected, rootGroup.getDistributionConfigHash()); + EXPECT_EQ(expected, rootGroup.getDistributionConfigHash()); } -} // lib -} // storage +} diff --git a/vdslib/src/tests/distribution/idealnodecalculatorimpltest.cpp b/vdslib/src/tests/distribution/idealnodecalculatorimpltest.cpp index b564a615994..f54c94ae8f9 100644 --- a/vdslib/src/tests/distribution/idealnodecalculatorimpltest.cpp +++ b/vdslib/src/tests/distribution/idealnodecalculatorimpltest.cpp @@ -1,22 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include #include -#include - -namespace storage { -namespace lib { - -struct IdealNodeCalculatorImplTest : public CppUnit::TestFixture { - - void testNormalUsage(); - - CPPUNIT_TEST_SUITE(IdealNodeCalculatorImplTest); - CPPUNIT_TEST(testNormalUsage); - CPPUNIT_TEST_SUITE_END(); -}; +#include +#include -CPPUNIT_TEST_SUITE_REGISTRATION(IdealNodeCalculatorImplTest); +namespace storage::lib { /** * Class is just a wrapper for distribution, so little needs to be tested. Just @@ -26,8 +14,7 @@ CPPUNIT_TEST_SUITE_REGISTRATION(IdealNodeCalculatorImplTest); * - Changes in distribution/cluster state is picked up. */ -void -IdealNodeCalculatorImplTest::testNormalUsage() +TEST(IdealNodeCalculatorImplTest, test_normal_usage) { ClusterState state("storage:10"); Distribution distr(Distribution::getDefaultDistributionConfig(3, 10)); @@ -38,10 +25,9 @@ IdealNodeCalculatorImplTest::testNormalUsage() configurable.setClusterState(state); std::string expected("[storage.8, storage.9, storage.6]"); - CPPUNIT_ASSERT_EQUAL( + EXPECT_EQ( expected, calc.getIdealStorageNodes(document::BucketId(16, 5)).toString()); } -} // lib -} // storage +} diff --git a/vdslib/src/tests/testrunner.cpp b/vdslib/src/tests/testrunner.cpp deleted file mode 100644 index 7c3fbfccc8b..00000000000 --- a/vdslib/src/tests/testrunner.cpp +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include - -#include -LOG_SETUP("vdslibcppunittestrunner"); - -int -main(int argc, const char *argv[]) -{ - vdstestlib::CppUnitTestRunner testRunner; - return testRunner.run(argc, argv); -} - -- cgit v1.2.3