diff options
6 files changed, 153 insertions, 25 deletions
diff --git a/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp index deb1b04e11a..b0f1220c768 100644 --- a/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp @@ -2,9 +2,12 @@ #include <vespa/searchcore/proton/documentmetastore/lid_allocator.h> #include <vespa/vespalib/util/generationholder.h> +#include <vespa/vespalib/util/time.h> #include <vespa/vespalib/gtest/gtest.h> +#include <iostream> using vespalib::GenerationHolder; +using vespalib::Timer; namespace proton { @@ -118,6 +121,42 @@ TEST_F(LidAllocatorTest, unregister_lids) EXPECT_EQ((std::vector<uint32_t>{1, 3, 5, 7, 8}), alloc_lids(5)); } +class LidAllocatorPerformanceTest : public LidAllocatorTest, + public testing::WithParamInterface<bool> +{ +}; + +TEST_P(LidAllocatorPerformanceTest, unregister_lids_performance) +{ + constexpr uint32_t test_size = 1000000; + _allocator.ensureSpace(test_size + 1, test_size + 1); + std::vector<std::vector<uint32_t>> buckets; + buckets.resize(1000); + auto reserve_size = (test_size + (buckets.size() - 1)) / buckets.size(); +for (auto& bucket : buckets) { + bucket.reserve(reserve_size); +} + for (uint32_t i = 0; i < test_size; ++i) { + _allocator.registerLid(i + 1); + buckets[i % buckets.size()].emplace_back(i + 1); + } + construct_free_list(); + Timer timer; + for (auto& bucket: buckets) { + if (GetParam()) { + unregister_lids(bucket); + } else { + for (auto lid : bucket) { + _allocator.unregisterLid(lid); + } + } + } + auto rate = test_size / vespalib::to_s(timer.elapsed()); + std::cout << "Unregister rate: " << std::fixed << rate << std::endl; +} + +VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(LidAllocatorParameterizedPerformanceTest, LidAllocatorPerformanceTest, testing::Values(false, true)); + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentmetastore/lid_state_vector/lid_state_vector_test.cpp b/searchcore/src/tests/proton/documentmetastore/lid_state_vector/lid_state_vector_test.cpp index af4c2efd74b..ab45cca0971 100644 --- a/searchcore/src/tests/proton/documentmetastore/lid_state_vector/lid_state_vector_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lid_state_vector/lid_state_vector_test.cpp @@ -141,6 +141,33 @@ TEST_F(LidStateVectorTest, lid_state_vector_resizing_is_working) assertLidStateVector({}, 2000, 0, lids); } +TEST_F(LidStateVectorTest, set_bits) +{ + LidStateVector lids(1000, 1000, _gen_hold, true, true); + EXPECT_EQ(100, lids.assert_not_set_bits({ 10, 40, 100 })); + assertLidStateVector({}, 1000, 0, lids); + EXPECT_EQ(100, lids.set_bits({ 10, 40, 100 })); + assertLidStateVector({ 10, 40, 100 }, 10, 100, lids); +} + +TEST_F(LidStateVectorTest, clear_bits) +{ + LidStateVector lids(1000, 1000, _gen_hold, true, true); + lids.set_bits({ 10, 40, 100 }); + lids.clear_bits({ 10, 100 }); + assertLidStateVector({ 40 }, 40, 40, lids); +} + +TEST_F(LidStateVectorTest, consider_clear_bits) +{ + LidStateVector lids(1000, 1000, _gen_hold, true, true); + lids.set_bits({ 40 }); + lids.consider_clear_bits({ 10, 100 }); + assertLidStateVector({ 40 }, 40, 40, lids); + lids.consider_clear_bits({ 10, 40, 100 }); + assertLidStateVector({}, 1000, 0, lids); +} + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index fc5ecfee48b..467d12581c2 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -82,9 +82,15 @@ LidAllocator::unregisterLid(DocId lid) void LidAllocator::unregister_lids(const std::vector<DocId>& lids) { - for (auto lid : lids) { - unregisterLid(lid); + if (lids.empty()) { + return; } + auto high = isFreeListConstructed() ? _pendingHoldLids.set_bits(lids) : _pendingHoldLids.assert_not_set_bits(lids); + assert(high < _usedLids.size()); + _usedLids.clear_bits(lids); + assert(high < _activeLids.size()); + _activeLids.consider_clear_bits(lids); + _numActiveLids = _activeLids.count(); } void diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp index 49e8d3eb23a..7309f7a518c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp @@ -90,6 +90,50 @@ LidStateVector::setBit(unsigned int idx) _bv.setBitAndMaintainCount(idx); } +template <bool do_set> +uint32_t +LidStateVector::assert_is_not_set_then_set_bits_helper(const std::vector<uint32_t>& idxs) +{ + uint32_t size = _bv.size(); + uint32_t high = 0; + uint32_t low = size; + for (auto idx : idxs) { + assert(idx < size); + if (idx > high) { + high = idx; + } + assert(!_bv.testBit(idx)); + if (do_set) { + if (idx < low) { + low = idx; + } + _bv.setBitAndMaintainCount(idx); + } + } + if (do_set) { + if (_trackLowest && low < _lowest) { + _lowest = low; + } + if (_trackHighest && high > _highest) { + _highest = high; + } + } + return high; +} + +uint32_t +LidStateVector::assert_not_set_bits(const std::vector<uint32_t>& idxs) +{ + return assert_is_not_set_then_set_bits_helper<false>(idxs); +} + +uint32_t +LidStateVector::set_bits(const std::vector<uint32_t>& idxs) +{ + return assert_is_not_set_then_set_bits_helper<true>(idxs); +} + + void LidStateVector::clearBit(unsigned int idx) { @@ -100,4 +144,30 @@ LidStateVector::clearBit(unsigned int idx) maybeUpdateHighest(); } +template <bool do_assert> +void +LidStateVector::assert_is_set_then_clear_bits_helper(const std::vector<uint32_t>& idxs) +{ + for (auto idx : idxs) { + if (do_assert) { + assert(_bv.testBit(idx)); + } + _bv.clearBitAndMaintainCount(idx); + } + maybeUpdateLowest(); + maybeUpdateHighest(); +} + +void +LidStateVector::consider_clear_bits(const std::vector<uint32_t>& idxs) +{ + assert_is_set_then_clear_bits_helper<false>(idxs); +} + +void +LidStateVector::clear_bits(const std::vector<uint32_t>& idxs) +{ + assert_is_set_then_clear_bits_helper<true>(idxs); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h index be47676716b..74851635124 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h @@ -25,6 +25,10 @@ class LidStateVector if (_trackHighest && _highest != 0 && !_bv.testBit(_highest)) updateHighest(); } + template <bool do_set> + uint32_t assert_is_not_set_then_set_bits_helper(const std::vector<uint32_t>& idxs); + template <bool do_assert> + void assert_is_set_then_clear_bits_helper(const std::vector<uint32_t>& idxs); public: LidStateVector(unsigned int newSize, unsigned int newCapacity, @@ -35,7 +39,11 @@ public: void resizeVector(uint32_t newSize, uint32_t newCapacity); void setBit(unsigned int idx); + uint32_t assert_not_set_bits(const std::vector<uint32_t>& idxs); + uint32_t set_bits(const std::vector<uint32_t>& idxs); void clearBit(unsigned int idx); + void consider_clear_bits(const std::vector<uint32_t>& idxs); + void clear_bits(const std::vector<uint32_t>& idxs); bool testBit(unsigned int idx) const { return _bv.testBit(idx); } unsigned int size() const { return _bv.size(); } unsigned int byteSize() const { diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index a03d215a6b2..60e3aff01cb 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -23,28 +23,6 @@ <maven.compiler.release>8</maven.compiler.release> </properties> - <!-- This is a client jar and should be compilable with jdk8 --> - <profiles> - <profile> - <id>jdk11</id> - <activation> - <jdk>11</jdk> - </activation> - <properties> - <java.version>11</java.version> - </properties> - </profile> - <profile> - <id>jdk1.8</id> - <activation> - <jdk>1.8</jdk> - </activation> - <properties> - <java.version>8</java.version> - </properties> - </profile> - </profiles> - <dependencies> <!-- Hadoop dependencies --> <dependency> @@ -211,7 +189,7 @@ <jdkToolchain> <version>${java.version}</version> </jdkToolchain> - <release>${java.version}</release> + <release>${maven.compiler.release}</release> </configuration> </plugin> </plugins> |