diff options
author | Geir Storli <geirst@yahooinc.com> | 2021-11-15 09:47:57 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-11-15 09:47:57 +0100 |
commit | 5d57b92c393693104d474c6eb06a23b5c5c2a0c8 (patch) | |
tree | 23bba02ba298c54c970daf9ba7dc6278c5017997 | |
parent | 5cb9b85ffb3ec2a77caaa4ee367ad6435bbf762a (diff) | |
parent | 3c4d5d5c23f69568b80b80858a53dd9d89ffe833 (diff) |
Merge pull request #20002 from vespa-engine/toregge/unit-test-lid-allocator-unregister-lids
Unit test proton::documentmetastore::LidAllocator::unregister_lids().
6 files changed, 291 insertions, 117 deletions
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 36c7e386445..e08b951d3f3 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -102,6 +102,8 @@ vespa_define_module( src/tests/proton/documentdb/threading_service_config src/tests/proton/documentmetastore src/tests/proton/documentmetastore/lidreusedelayer + src/tests/proton/documentmetastore/lid_allocator + src/tests/proton/documentmetastore/lid_state_vector src/tests/proton/feed_and_search src/tests/proton/feedoperation src/tests/proton/feedtoken diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index d201b02a61d..579633bfaa2 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -378,123 +378,6 @@ TEST(DocumentMetaStore, generation_handling_is_working) EXPECT_EQ(3u, gh.getCurrentGeneration()); } -TEST(DocumentMetaStoreTest, basic_free_list_is_working) -{ - GenerationHolder genHold; - LidStateVector freeLids(100, 100, genHold, true, false); - LidHoldList list; - EXPECT_TRUE(freeLids.empty()); - EXPECT_EQ(0u, freeLids.count()); - EXPECT_EQ(0u, list.size()); - - list.add(10, 10); - EXPECT_TRUE(freeLids.empty()); - EXPECT_EQ(0u, freeLids.count()); - EXPECT_EQ(1u, list.size()); - - list.add(20, 20); - list.add(30, 30); - EXPECT_TRUE(freeLids.empty()); - EXPECT_EQ(0u, freeLids.count()); - EXPECT_EQ(3u, list.size()); - - list.trimHoldLists(20, freeLids); - EXPECT_FALSE(freeLids.empty()); - EXPECT_EQ(1u, freeLids.count()); - - EXPECT_EQ(10u, freeLids.getLowest()); - freeLids.clearBit(10); - EXPECT_TRUE(freeLids.empty()); - EXPECT_EQ(0u, freeLids.count()); - EXPECT_EQ(2u, list.size()); - - list.trimHoldLists(31, freeLids); - EXPECT_FALSE(freeLids.empty()); - EXPECT_EQ(2u, freeLids.count()); - - EXPECT_EQ(20u, freeLids.getLowest()); - freeLids.clearBit(20); - EXPECT_FALSE(freeLids.empty()); - EXPECT_EQ(1u, freeLids.count()); - EXPECT_EQ(0u, list.size()); - - EXPECT_EQ(30u, freeLids.getLowest()); - freeLids.clearBit(30); - EXPECT_TRUE(freeLids.empty()); - EXPECT_EQ(0u, list.size()); - EXPECT_EQ(0u, freeLids.count()); -} - -void -assertLidStateVector(const std::vector<uint32_t> &expLids, uint32_t lowest, uint32_t highest, - const LidStateVector &actLids) -{ - if (!expLids.empty()) { - EXPECT_EQ(expLids.size(), actLids.count()); - uint32_t trueBit = 0; - for (auto i : expLids) { - EXPECT_TRUE(actLids.testBit(i)); - trueBit = actLids.getNextTrueBit(trueBit); - EXPECT_EQ(i, trueBit); - ++trueBit; - } - trueBit = actLids.getNextTrueBit(trueBit); - EXPECT_EQ(actLids.size(), trueBit); - EXPECT_EQ(lowest, actLids.getLowest()); - EXPECT_EQ(highest, actLids.getHighest()); - } else { - EXPECT_TRUE(actLids.empty()); - } -} - -TEST(DocumentMetaStoreTest, lid_state_vector_resizing_is_working) -{ - GenerationHolder genHold; - LidStateVector lids(1000, 1000, genHold, true, true); - lids.setBit(3); - lids.setBit(150); - lids.setBit(270); - lids.setBit(310); - lids.setBit(440); - lids.setBit(780); - lids.setBit(930); - assertLidStateVector({3,150,270,310,440,780,930}, 3, 930, lids); - - lids.resizeVector(1500, 1500); - assertLidStateVector({3,150,270,310,440,780,930}, 3, 930, lids); - lids.clearBit(3); - assertLidStateVector({150,270,310,440,780,930}, 150, 930, lids); - lids.clearBit(150); - assertLidStateVector({270,310,440,780,930}, 270, 930, lids); - lids.setBit(170); - assertLidStateVector({170,270,310,440,780,930}, 170, 930, lids); - lids.setBit(1490); - assertLidStateVector({170,270,310,440,780,930,1490}, 170, 1490, lids); - - lids.resizeVector(2000, 2000); - assertLidStateVector({170,270,310,440,780,930,1490}, 170, 1490, lids); - lids.clearBit(170); - assertLidStateVector({270,310,440,780,930,1490}, 270, 1490, lids); - lids.clearBit(270); - assertLidStateVector({310,440,780,930,1490}, 310, 1490, lids); - lids.setBit(1990); - assertLidStateVector({310,440,780,930,1490,1990}, 310, 1990, lids); - lids.clearBit(310); - assertLidStateVector({440,780,930,1490,1990}, 440, 1990, lids); - lids.clearBit(440); - assertLidStateVector({780,930,1490,1990}, 780, 1990, lids); - lids.clearBit(780); - assertLidStateVector({930,1490,1990}, 930, 1990, lids); - lids.clearBit(930); - assertLidStateVector({1490,1990}, 1490, 1990, lids); - lids.clearBit(1490); - assertLidStateVector({1990}, 1990, 1990, lids); - lids.clearBit(1990); - assertLidStateVector({}, 0, 0, lids); - - genHold.clearHoldLists(); -} - TEST(DocumentMetaStoreTest, lid_and_gid_space_is_reused) { auto dms = std::make_shared<DocumentMetaStore>(createBucketDB()); diff --git a/searchcore/src/tests/proton/documentmetastore/lid_allocator/CMakeLists.txt b/searchcore/src/tests/proton/documentmetastore/lid_allocator/CMakeLists.txt new file mode 100644 index 00000000000..8fc5c23239f --- /dev/null +++ b/searchcore/src/tests/proton/documentmetastore/lid_allocator/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_lid_allocator_test_app TEST + SOURCES + lid_allocator_test.cpp + DEPENDS + searchcore_documentmetastore + GTest::GTest +) +vespa_add_test(NAME searchcore_lid_allocator_test_app COMMAND searchcore_lid_allocator_test_app) + diff --git a/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp new file mode 100644 index 00000000000..deb1b04e11a --- /dev/null +++ b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp @@ -0,0 +1,123 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/documentmetastore/lid_allocator.h> +#include <vespa/vespalib/util/generationholder.h> +#include <vespa/vespalib/gtest/gtest.h> + +using vespalib::GenerationHolder; + +namespace proton { + +using documentmetastore::LidAllocator; + +class LidAllocatorTest : public ::testing::Test +{ +protected: + GenerationHolder _gen_hold; + LidAllocator _allocator; + + LidAllocatorTest() + : ::testing::Test(), + _gen_hold(), + _allocator(100, 100, _gen_hold) + { + } + + ~LidAllocatorTest() + { + _gen_hold.clearHoldLists(); + } + + uint32_t get_size() { return _allocator.getActiveLids().size(); } + + void construct_free_list() { + _allocator.constructFreeList(_allocator.getActiveLids().size()); + _allocator.setFreeListConstructed(); + } + + void register_lids(const std::vector<uint32_t>& lids) { + for (uint32_t lid : lids) { + _allocator.registerLid(lid); + } + } + + std::vector<uint32_t> alloc_lids(uint32_t count) { + std::vector<uint32_t> result; + for (uint32_t i = 0; i < count; ++i) { + result.emplace_back(_allocator.getFreeLid(get_size())); + } + return result; + } + + void activate_lids(const std::vector<uint32_t>& lids, bool active) { + for (uint32_t lid : lids) { + _allocator.updateActiveLids(lid, active); + } + } + + void unregister_lids(const std::vector<uint32_t>& lids) { + _allocator.unregister_lids(lids); + } + + void hold_lids(const std::vector<uint32_t>& lids) { + _allocator.holdLids(lids, get_size(), 0); + } + + void trim_hold_lists() { + _allocator.trimHoldLists(1); + } + + std::vector<uint32_t> get_valid_lids() { + std::vector<uint32_t> result; + auto size = get_size(); + for (uint32_t lid = 1; lid < size; ++lid) { + if (_allocator.validLid(lid)) { + result.emplace_back(lid); + } + } + return result; + } + + std::vector<uint32_t> get_active_lids() { + std::vector<uint32_t> result; + auto active_lids = _allocator.getActiveLids(); + uint32_t lid = active_lids.getNextTrueBit(1); + while (lid < active_lids.size()) { + if (active_lids.testBit(lid)) { + result.emplace_back(lid); + } + lid = active_lids.getNextTrueBit(lid + 1); + } + return result; + } + + void + assert_valid_lids(const std::vector<uint32_t>& exp_lids) { + EXPECT_EQ(exp_lids, get_valid_lids()); + } + + void + assert_active_lids(const std::vector<uint32_t>& exp_lids) { + EXPECT_EQ(exp_lids, get_active_lids()); + } + +}; + +TEST_F(LidAllocatorTest, unregister_lids) +{ + register_lids({ 1, 2, 3, 4, 5, 6 }); + activate_lids({ 4, 5, 6 }, true); + assert_valid_lids({1, 2, 3, 4, 5, 6}); + assert_active_lids({4, 5, 6}); + construct_free_list(); + unregister_lids({1, 3, 5}); + assert_valid_lids({2, 4, 6}); + assert_active_lids({4, 6}); + hold_lids({1, 3, 5}); + trim_hold_lists(); + EXPECT_EQ((std::vector<uint32_t>{1, 3, 5, 7, 8}), alloc_lids(5)); +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentmetastore/lid_state_vector/CMakeLists.txt b/searchcore/src/tests/proton/documentmetastore/lid_state_vector/CMakeLists.txt new file mode 100644 index 00000000000..652e83a6b79 --- /dev/null +++ b/searchcore/src/tests/proton/documentmetastore/lid_state_vector/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_lid_state_vector_test_app TEST + SOURCES + lid_state_vector_test.cpp + DEPENDS + searchcore_documentmetastore + GTest::GTest +) +vespa_add_test(NAME searchcore_lid_state_vector_test_app COMMAND searchcore_lid_state_vector_test_app) + diff --git a/searchcore/src/tests/proton/documentmetastore/lid_state_vector/lid_state_vector_test.cpp b/searchcore/src/tests/proton/documentmetastore/lid_state_vector/lid_state_vector_test.cpp new file mode 100644 index 00000000000..af4c2efd74b --- /dev/null +++ b/searchcore/src/tests/proton/documentmetastore/lid_state_vector/lid_state_vector_test.cpp @@ -0,0 +1,146 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/documentmetastore/lidstatevector.h> +#include <vespa/searchcore/proton/documentmetastore/lid_hold_list.h> +#include <vespa/vespalib/util/generationholder.h> +#include <vespa/vespalib/gtest/gtest.h> + +using vespalib::GenerationHolder; + +namespace proton { + +class LidStateVectorTest : public ::testing::Test +{ +protected: + GenerationHolder _gen_hold; + + LidStateVectorTest() + : ::testing::Test(), + _gen_hold() + { + } + + ~LidStateVectorTest() + { + _gen_hold.clearHoldLists(); + } + +}; + + +TEST_F(LidStateVectorTest, basic_free_list_is_working) +{ + LidStateVector freeLids(100, 100, _gen_hold, true, false); + LidHoldList list; + EXPECT_TRUE(freeLids.empty()); + EXPECT_EQ(0u, freeLids.count()); + EXPECT_EQ(0u, list.size()); + + list.add(10, 10); + EXPECT_TRUE(freeLids.empty()); + EXPECT_EQ(0u, freeLids.count()); + EXPECT_EQ(1u, list.size()); + + list.add(20, 20); + list.add(30, 30); + EXPECT_TRUE(freeLids.empty()); + EXPECT_EQ(0u, freeLids.count()); + EXPECT_EQ(3u, list.size()); + + list.trimHoldLists(20, freeLids); + EXPECT_FALSE(freeLids.empty()); + EXPECT_EQ(1u, freeLids.count()); + + EXPECT_EQ(10u, freeLids.getLowest()); + freeLids.clearBit(10); + EXPECT_TRUE(freeLids.empty()); + EXPECT_EQ(0u, freeLids.count()); + EXPECT_EQ(2u, list.size()); + + list.trimHoldLists(31, freeLids); + EXPECT_FALSE(freeLids.empty()); + EXPECT_EQ(2u, freeLids.count()); + + EXPECT_EQ(20u, freeLids.getLowest()); + freeLids.clearBit(20); + EXPECT_FALSE(freeLids.empty()); + EXPECT_EQ(1u, freeLids.count()); + EXPECT_EQ(0u, list.size()); + + EXPECT_EQ(30u, freeLids.getLowest()); + freeLids.clearBit(30); + EXPECT_TRUE(freeLids.empty()); + EXPECT_EQ(0u, list.size()); + EXPECT_EQ(0u, freeLids.count()); +} + +void +assertLidStateVector(const std::vector<uint32_t> &expLids, uint32_t lowest, uint32_t highest, + const LidStateVector &actLids) +{ + if (!expLids.empty()) { + EXPECT_EQ(expLids.size(), actLids.count()); + uint32_t trueBit = 0; + for (auto i : expLids) { + EXPECT_TRUE(actLids.testBit(i)); + trueBit = actLids.getNextTrueBit(trueBit); + EXPECT_EQ(i, trueBit); + ++trueBit; + } + trueBit = actLids.getNextTrueBit(trueBit); + EXPECT_EQ(actLids.size(), trueBit); + } else { + EXPECT_TRUE(actLids.empty()); + } + EXPECT_EQ(lowest, actLids.getLowest()); + EXPECT_EQ(highest, actLids.getHighest()); +} + +TEST_F(LidStateVectorTest, lid_state_vector_resizing_is_working) +{ + LidStateVector lids(1000, 1000, _gen_hold, true, true); + lids.setBit(3); + lids.setBit(150); + lids.setBit(270); + lids.setBit(310); + lids.setBit(440); + lids.setBit(780); + lids.setBit(930); + assertLidStateVector({3,150,270,310,440,780,930}, 3, 930, lids); + + lids.resizeVector(1500, 1500); + assertLidStateVector({3,150,270,310,440,780,930}, 3, 930, lids); + lids.clearBit(3); + assertLidStateVector({150,270,310,440,780,930}, 150, 930, lids); + lids.clearBit(150); + assertLidStateVector({270,310,440,780,930}, 270, 930, lids); + lids.setBit(170); + assertLidStateVector({170,270,310,440,780,930}, 170, 930, lids); + lids.setBit(1490); + assertLidStateVector({170,270,310,440,780,930,1490}, 170, 1490, lids); + + lids.resizeVector(2000, 2000); + assertLidStateVector({170,270,310,440,780,930,1490}, 170, 1490, lids); + lids.clearBit(170); + assertLidStateVector({270,310,440,780,930,1490}, 270, 1490, lids); + lids.clearBit(270); + assertLidStateVector({310,440,780,930,1490}, 310, 1490, lids); + lids.setBit(1990); + assertLidStateVector({310,440,780,930,1490,1990}, 310, 1990, lids); + lids.clearBit(310); + assertLidStateVector({440,780,930,1490,1990}, 440, 1990, lids); + lids.clearBit(440); + assertLidStateVector({780,930,1490,1990}, 780, 1990, lids); + lids.clearBit(780); + assertLidStateVector({930,1490,1990}, 930, 1990, lids); + lids.clearBit(930); + assertLidStateVector({1490,1990}, 1490, 1990, lids); + lids.clearBit(1490); + assertLidStateVector({1990}, 1990, 1990, lids); + lids.clearBit(1990); + assertLidStateVector({}, 2000, 0, lids); +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() |