diff options
9 files changed, 25 insertions, 51 deletions
diff --git a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h index 8a9b94f3af8..65a7b5a276c 100644 --- a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h +++ b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h @@ -32,6 +32,20 @@ public: _maxDeadAddressSpaceRatio == rhs._maxDeadAddressSpaceRatio; } bool operator!=(const CompactionStrategy & rhs) const { return !(operator==(rhs)); } + + static constexpr size_t DEAD_BYTES_SLACK = 0x10000u; + + bool should_compact_memory(size_t used_bytes, size_t dead_bytes) const { + return ((dead_bytes >= DEAD_BYTES_SLACK) && + (dead_bytes > used_bytes * getMaxDeadBytesRatio())); + } + + static constexpr size_t DEAD_ADDRESS_SPACE_SLACK = 0x10000u; + + bool should_compact_address_space(size_t used_address_space, size_t dead_address_space) const { + return ((dead_address_space >= DEAD_ADDRESS_SPACE_SLACK) && + (dead_address_space > used_address_space * getMaxDeadAddressSpaceRatio())); + } }; std::ostream& operator<<(std::ostream& os, const CompactionStrategy& compaction_strategy); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index d0cada2224f..23481aa416a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -199,13 +199,6 @@ DocumentMetaStore::insert(GidToLidMapKey key, const RawDocumentMetaData &metaDat updateCommittedDocIdLimit(); } -namespace { - -// minimum dead bytes in gid to lid map before consider compaction -constexpr size_t DEAD_BYTES_SLACK = 0x10000u; - -} - bool DocumentMetaStore::consider_compact_gid_to_lid_map() { @@ -215,9 +208,7 @@ DocumentMetaStore::consider_compact_gid_to_lid_map() auto &compaction_strategy = getConfig().getCompactionStrategy(); size_t used_bytes = _cached_gid_to_lid_map_memory_usage.usedBytes(); size_t dead_bytes = _cached_gid_to_lid_map_memory_usage.deadBytes(); - bool compact_memory = ((dead_bytes >= DEAD_BYTES_SLACK) && - (used_bytes * compaction_strategy.getMaxDeadBytesRatio() < dead_bytes)); - return compact_memory; + return compaction_strategy.should_compact_memory(used_bytes, dead_bytes); } void diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp index c3bb9481f0f..a029373821a 100644 --- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp @@ -237,8 +237,7 @@ TEST_F("Compaction limits address space usage (dead) when free lists are NOT use { populate_and_hammer(f, true); AddressSpace afterSpace = f.getMultiValueAddressSpaceUsage("after"); - // DEAD_ARRAYS_SLACK in multi value mapping is is 64k - EXPECT_GREATER(65536u, afterSpace.dead()); + EXPECT_GREATER(search::CompactionStrategy::DEAD_ADDRESS_SPACE_SLACK, afterSpace.dead()); } TEST_F("Compaction is not executed when free lists are used", @@ -267,8 +266,7 @@ TEST_F("Compaction is peformed when compaction strategy is changed to enable com f._v->commit(); // new commit might trigger further compaction after2 = f.getMultiValueAddressSpaceUsage("after2"); } - // DEAD_ARRAYS_SLACK in multi value mapping is is 64k - EXPECT_GREATER(65536u, after2.dead()); + EXPECT_GREATER(search::CompactionStrategy::DEAD_ADDRESS_SPACE_SLACK, after2.dead()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp b/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp index bd1b63c01d8..2faaa371fca 100644 --- a/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp @@ -174,9 +174,8 @@ template <typename VectorType> void CompactionTest<VectorType>::test_enum_store_compaction() { - constexpr size_t DEAD_BYTES_SLACK = 0x10000u; constexpr uint32_t canary_stride = 256; - uint32_t dead_limit = DEAD_BYTES_SLACK / 8; + uint32_t dead_limit = search::CompactionStrategy::DEAD_BYTES_SLACK / 8; uint32_t doc_count = dead_limit * 3; if (_v->hasMultiValue() || std::is_same_v<VectorType,StringAttribute>) { doc_count /= 2; diff --git a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp index a8a34b0ba8a..f45c3e4a1a0 100644 --- a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp +++ b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp @@ -45,8 +45,6 @@ vespalib::string doc1("id:test:music::1"); vespalib::string doc2("id:test:music::2"); vespalib::string doc3("id:test:music::3"); -constexpr size_t DEAD_BYTES_SLACK = 0x10000u; - } struct MyGidToLidMapperFactory : public search::attribute::test::MockGidToLidMapperFactory { @@ -178,7 +176,7 @@ struct ReferenceAttributeTest : public ::testing::Test { search::attribute::Status newStatus = oldStatus; uint64_t iter = 0; AttributeGuard guard(_attr); - uint64_t dropCount = DEAD_BYTES_SLACK / sizeof(Reference); + uint64_t dropCount = search::CompactionStrategy::DEAD_BYTES_SLACK / sizeof(Reference); for (; iter < iterLimit; ++iter) { clear(2); set(2, toGid(doc2)); diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index de68e7f044a..99d9a7fdfee 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -219,13 +219,6 @@ EnumStoreT<EntryT>::update_stat() return retval; } -namespace { - -// minimum dead bytes in enum store before consider compaction -constexpr size_t DEAD_BYTES_SLACK = 0x10000u; -constexpr size_t DEAD_ADDRESS_SPACE_SLACK = 0x10000u; - -} template <typename EntryT> std::unique_ptr<IEnumStore::EnumIndexRemapper> EnumStoreT<EntryT>::consider_compact(const CompactionStrategy& compaction_strategy) @@ -234,10 +227,8 @@ EnumStoreT<EntryT>::consider_compact(const CompactionStrategy& compaction_strate size_t dead_bytes = _cached_values_memory_usage.deadBytes(); size_t used_address_space = _cached_values_address_space_usage.used(); size_t dead_address_space = _cached_values_address_space_usage.dead(); - bool compact_memory = ((dead_bytes >= DEAD_BYTES_SLACK) && - (used_bytes * compaction_strategy.getMaxDeadBytesRatio() < dead_bytes)); - bool compact_address_space = ((dead_address_space >= DEAD_ADDRESS_SPACE_SLACK) && - (used_address_space * compaction_strategy.getMaxDeadAddressSpaceRatio() < dead_address_space)); + bool compact_memory = compaction_strategy.should_compact_memory(used_bytes, dead_bytes); + bool compact_address_space = compaction_strategy.should_compact_address_space(used_address_space, dead_address_space); if (compact_memory || compact_address_space) { return compact_worst(compact_memory, compact_address_space); } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp index b8378b059f6..c57c18e1b0e 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp @@ -6,14 +6,6 @@ namespace search::attribute { -namespace { - -// minimum dead bytes in multi value mapping before consider compaction -constexpr size_t DEAD_BYTES_SLACK = 0x10000u; -constexpr size_t DEAD_ARRAYS_SLACK = 0x10000u; - -} - MultiValueMappingBase::MultiValueMappingBase(const vespalib::GrowStrategy &gs, vespalib::GenerationHolder &genHolder) : _indices(gs, genHolder), @@ -89,10 +81,8 @@ MultiValueMappingBase::considerCompact(const CompactionStrategy &compactionStrat size_t deadBytes = _cachedArrayStoreMemoryUsage.deadBytes(); size_t usedArrays = _cachedArrayStoreAddressSpaceUsage.used(); size_t deadArrays = _cachedArrayStoreAddressSpaceUsage.dead(); - bool compactMemory = ((deadBytes >= DEAD_BYTES_SLACK) && - (usedBytes * compactionStrategy.getMaxDeadBytesRatio() < deadBytes)); - bool compactAddressSpace = ((deadArrays >= DEAD_ARRAYS_SLACK) && - (usedArrays * compactionStrategy.getMaxDeadAddressSpaceRatio() < deadArrays)); + bool compactMemory = compactionStrategy.should_compact_memory(usedBytes, deadBytes); + bool compactAddressSpace = compactionStrategy.should_compact_address_space(usedArrays, deadArrays); if (compactMemory || compactAddressSpace) { compactWorst(compactMemory, compactAddressSpace); return true; diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index 2af5bfdf225..f9faed34d90 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -27,9 +27,6 @@ using document::IdParseException; namespace { -// minimum dead bytes in unique store before consider compaction -constexpr size_t DEAD_BYTES_SLACK = 0x10000u; - const vespalib::string uniqueValueCountTag = "uniqueValueCount"; uint64_t @@ -289,8 +286,7 @@ ReferenceAttribute::considerCompact(const CompactionStrategy &compactionStrategy { size_t usedBytes = _cached_unique_store_values_memory_usage.usedBytes(); size_t deadBytes = _cached_unique_store_values_memory_usage.deadBytes(); - bool compactMemory = ((deadBytes >= DEAD_BYTES_SLACK) && - (usedBytes * compactionStrategy.getMaxDeadBytesRatio() < deadBytes)); + bool compactMemory = compactionStrategy.should_compact_memory(usedBytes, deadBytes); if (compactMemory) { compactWorst(); return true; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 042ecba0901..e8a25e0e23e 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -27,9 +27,6 @@ namespace { constexpr uint32_t TENSOR_ATTRIBUTE_VERSION = 0; -// minimum dead bytes in tensor attribute before consider compaction -constexpr size_t DEAD_SLACK = 0x10000u; - Value::UP createEmptyTensor(const ValueType &type) { @@ -90,7 +87,7 @@ TensorAttribute::onCommit() Status &status = getStatus(); size_t used = status.getUsed(); size_t dead = status.getDead(); - if ((dead >= DEAD_SLACK) && (dead * 5 > used)) { + if (getConfig().getCompactionStrategy().should_compact_memory(used, dead)) { compactWorst(); } } |