From 7aef2017e0af21f9b0f1290f2299bcb39319a384 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 24 May 2022 09:07:00 +0000 Subject: - Introduce the concept of minimal capacity for rcu vectors. - This is to prevent shrinking down past what has been configured in proton.documentdb[].allocation.initialnumdocs - shrinking down past the configured value is not supported as that would counter the preallocation done to prevent resizing. --- .../searchcommon/attribute/config/attribute_config_test.cpp | 3 ++- searchlib/src/vespa/searchcommon/common/growstrategy.cpp | 4 +++- searchlib/src/vespa/searchcommon/common/growstrategy.h | 12 ++++++++---- searchlib/src/vespa/searchlib/attribute/attributevector.h | 4 +--- searchlib/src/vespa/searchlib/attribute/changevector.h | 1 + .../src/vespa/searchlib/attribute/reference_mappings.cpp | 4 ++-- searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp | 2 +- 7 files changed, 18 insertions(+), 12 deletions(-) (limited to 'searchlib') diff --git a/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp index 918e14546e6..8abe5dcc414 100644 --- a/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp +++ b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp @@ -101,11 +101,12 @@ TEST("test operator== on attribute config for tensor type") } TEST("Test GrowStrategy consistency") { - GrowStrategy g(1024, 0.5, 17, 0.4f); + GrowStrategy g(1024, 0.5, 17, 3, 0.4f); EXPECT_EQUAL(1024u, g.getDocsInitialCapacity()); EXPECT_EQUAL(50u, g.getDocsGrowPercent()); EXPECT_EQUAL(0.5, g.getDocsGrowFactor()); EXPECT_EQUAL(17u, g.getDocsGrowDelta()); + EXPECT_EQUAL(3u, g.getDocsMinimumCapacity()); EXPECT_EQUAL(0.4f, g.getMultiValueAllocGrowFactor()); } diff --git a/searchlib/src/vespa/searchcommon/common/growstrategy.cpp b/searchlib/src/vespa/searchcommon/common/growstrategy.cpp index f35cdbaa640..36c577b1048 100644 --- a/searchlib/src/vespa/searchcommon/common/growstrategy.cpp +++ b/searchlib/src/vespa/searchcommon/common/growstrategy.cpp @@ -5,9 +5,11 @@ namespace search { -std::ostream& operator<<(std::ostream& os, const GrowStrategy& grow_strategy) +std::ostream& +operator<<(std::ostream& os, const GrowStrategy& grow_strategy) { os << "{docsInitialCapacity=" << grow_strategy.getDocsInitialCapacity() << + ", docsMinimumCapacity=" << grow_strategy.getDocsMinimumCapacity() << ", docsGrowFactor=" << grow_strategy.getDocsGrowFactor() << ", docsGrowDelta=" << grow_strategy.getDocsGrowDelta() << ", multiValueAllocGrowFactor=" << grow_strategy.getMultiValueAllocGrowFactor() << diff --git a/searchlib/src/vespa/searchcommon/common/growstrategy.h b/searchlib/src/vespa/searchcommon/common/growstrategy.h index b9b4a42cf72..3320e6ead6e 100644 --- a/searchlib/src/vespa/searchcommon/common/growstrategy.h +++ b/searchlib/src/vespa/searchcommon/common/growstrategy.h @@ -12,16 +12,18 @@ class GrowStrategy { private: uint32_t _docsInitialCapacity; + uint32_t _docsMinimumCapacity; float _docsGrowFactor; uint32_t _docsGrowDelta; float _multiValueAllocGrowFactor; public: GrowStrategy() noexcept - : GrowStrategy(1024, 0.5, 0, 0.2) + : GrowStrategy(1024, 0.5, 0, 0, 0.2) {} GrowStrategy(uint32_t docsInitialCapacity, float docsGrowFactor, - uint32_t docsGrowDelta, float multiValueAllocGrowFactor) noexcept + uint32_t docsGrowDelta, uint32_t docsMinimumCapacity, float multiValueAllocGrowFactor) noexcept : _docsInitialCapacity(docsInitialCapacity), + _docsMinimumCapacity(docsMinimumCapacity), _docsGrowFactor(docsGrowFactor), _docsGrowDelta(docsGrowDelta), _multiValueAllocGrowFactor(multiValueAllocGrowFactor) @@ -29,19 +31,21 @@ public: } static GrowStrategy make(uint32_t docsInitialCapacity, float docsGrowFactor, uint32_t docsGrowDelta) { - return GrowStrategy(docsInitialCapacity, docsGrowFactor, docsGrowDelta, 0.2); + return GrowStrategy(docsInitialCapacity, docsGrowFactor, docsGrowDelta, 0, 0.2); } uint32_t getDocsInitialCapacity() const { return _docsInitialCapacity; } + uint32_t getDocsMinimumCapacity() const { return _docsMinimumCapacity; } uint32_t getDocsGrowPercent() const { return _docsGrowFactor*100; } float getDocsGrowFactor() const { return _docsGrowFactor; } uint32_t getDocsGrowDelta() const { return _docsGrowDelta; } float getMultiValueAllocGrowFactor() const { return _multiValueAllocGrowFactor; } void setDocsInitialCapacity(uint32_t v) { _docsInitialCapacity = v; } + void setDocsMinimumCapacity(uint32_t v) { _docsMinimumCapacity = v; } void setDocsGrowDelta(uint32_t v) { _docsGrowDelta = v; } vespalib::GrowStrategy to_generic_strategy() const { - return vespalib::GrowStrategy(_docsInitialCapacity, _docsGrowFactor, _docsGrowDelta); + return vespalib::GrowStrategy(_docsInitialCapacity, _docsGrowFactor, _docsGrowDelta, _docsMinimumCapacity); } bool operator==(const GrowStrategy & rhs) const { diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 57c73067671..09dc27f0451 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -5,7 +5,6 @@ #include "address_space_usage.h" #include "changevector.h" #include "readable_attribute_vector.h" -#include #include #include #include @@ -19,8 +18,7 @@ #include #include #include -#include -#include +#include #include #include #include diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.h b/searchlib/src/vespa/searchlib/attribute/changevector.h index d929e8615a9..5f858a3e012 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.h +++ b/searchlib/src/vespa/searchlib/attribute/changevector.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include namespace vespalib { class MemoryUsage; } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp index fe61913b801..37b7a82ab55 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp @@ -9,10 +9,10 @@ namespace search::attribute { ReferenceMappings::ReferenceMappings(GenerationHolder &genHolder, const std::atomic& committedDocIdLimit) - : _reverseMappingIndices(vespalib::GrowStrategy(16, 1.0, 0), genHolder), + : _reverseMappingIndices(vespalib::GrowStrategy(16, 1.0, 0, 0), genHolder), _targetLidLimit(0), _reverseMapping(), - _targetLids(vespalib::GrowStrategy(16, 1.0, 0), genHolder), + _targetLids(vespalib::GrowStrategy(16, 1.0, 0, 0), genHolder), _committedDocIdLimit(committedDocIdLimit) { } diff --git a/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp b/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp index 74df5c8c5d4..15e09172718 100644 --- a/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp +++ b/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp @@ -17,7 +17,7 @@ class CondensedBitVectorT : public CondensedBitVector { public: CondensedBitVectorT(size_t sz, GenerationHolder &genHolder) : - _v(vespalib::GrowStrategy(sz, 30, 1000), genHolder) + _v(vespalib::GrowStrategy(sz, 30, 1000, 0), genHolder) { for (size_t i = 0; i < sz; ++i) { _v.push_back(0); -- cgit v1.2.3