diff options
39 files changed, 166 insertions, 76 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 3170654409b..f2e8ce10e7c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -209,7 +209,7 @@ DocumentMetaStore::onCommit() if (consider_compact_gid_to_lid_map()) { incGeneration(); _changesSinceCommit = 0; - _gidToLidMap.compact_worst(); + _gidToLidMap.compact_worst(getConfig().getCompactionStrategy()); _gid_to_lid_map_write_itr_prepare_serial_num = 0u; _gid_to_lid_map_write_itr.begin(_gidToLidMap.getRoot()); incGeneration(); diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp index 8b1906573d4..bddaa4f4e31 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp @@ -14,6 +14,8 @@ LOG_SETUP("multivaluemapping_test"); using vespalib::datastore::ArrayStoreConfig; +using vespalib::datastore::CompactionSpec; +using vespalib::datastore::CompactionStrategy; template <typename EntryT> void @@ -142,7 +144,9 @@ public: } void compactWorst() { - _mvMapping->compactWorst(true, false); + CompactionSpec compaction_spec(true, false); + CompactionStrategy compaction_strategy; + _mvMapping->compactWorst(compaction_spec, compaction_strategy); _attr->commit(); _attr->incGeneration(); } diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index c35aba92c56..bb2d750eade 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/tensor/hnsw_index.h> #include <vespa/searchlib/tensor/random_level_generator.h> #include <vespa/searchlib/tensor/inv_log_level_generator.h> +#include <vespa/vespalib/datastore/compaction_spec.h> #include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/generationhandler.h> @@ -21,6 +22,7 @@ using namespace search::tensor; using namespace vespalib::slime; using vespalib::Slime; using search::BitVector; +using vespalib::datastore::CompactionSpec; using vespalib::datastore::CompactionStrategy; template <typename FloatType> @@ -628,8 +630,10 @@ TEST_F(HnswIndexTest, hnsw_graph_is_compacted) for (uint32_t i = 0; i < 10; ++i) { mem_1 = mem_2; // Forced compaction to move things around - index->compact_link_arrays(true, false); - index->compact_level_arrays(true, false); + CompactionSpec compaction_spec(true, false); + CompactionStrategy compaction_strategy; + index->compact_link_arrays(compaction_spec, compaction_strategy); + index->compact_level_arrays(compaction_spec, compaction_strategy); commit(); index->update_stat(); mem_2 = commit_and_update_stat(); diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index a140a529c7d..12871c967ff 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -201,7 +201,7 @@ public: void free_unused_values(IndexList to_remove); vespalib::MemoryUsage update_stat() override; std::unique_ptr<EnumIndexRemapper> consider_compact_values(const CompactionStrategy& compaction_strategy) override; - std::unique_ptr<EnumIndexRemapper> compact_worst_values(bool compact_memory, bool compact_address_space) override; + std::unique_ptr<EnumIndexRemapper> compact_worst_values(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) override; bool consider_compact_dictionary(const CompactionStrategy& compaction_strategy) override; uint64_t get_compaction_count() const override { return _store.get_data_store().get_compaction_count(); diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 897a2153b2b..2c985d2edf5 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -236,16 +236,17 @@ EnumStoreT<EntryT>::consider_compact_values(const CompactionStrategy& compaction bool compact_memory = compaction_strategy.should_compact_memory(used_bytes, dead_bytes); bool compact_address_space = compaction_strategy.should_compact_address_space(used_address_space, dead_address_space); if (compact_memory || compact_address_space) { - return compact_worst_values(compact_memory, compact_address_space); + CompactionSpec compaction_spec(compact_memory, compact_address_space); + return compact_worst_values(compaction_spec, compaction_strategy); } return std::unique_ptr<IEnumStore::EnumIndexRemapper>(); } template <typename EntryT> std::unique_ptr<IEnumStore::EnumIndexRemapper> -EnumStoreT<EntryT>::compact_worst_values(bool compact_memory, bool compact_address_space) +EnumStoreT<EntryT>::compact_worst_values(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - return _store.compact_worst(compact_memory, compact_address_space); + return _store.compact_worst(compaction_spec, compaction_strategy); } template <typename EntryT> @@ -258,13 +259,13 @@ EnumStoreT<EntryT>::consider_compact_dictionary(const CompactionStrategy& compac if (compaction_strategy.should_compact_memory(_cached_dictionary_btree_usage.usedBytes(), _cached_dictionary_btree_usage.deadBytes())) { - _dict->compact_worst(true, false); + _dict->compact_worst(true, false, compaction_strategy); return true; } if (compaction_strategy.should_compact_memory(_cached_dictionary_hash_usage.usedBytes(), _cached_dictionary_hash_usage.deadBytes())) { - _dict->compact_worst(false, true); + _dict->compact_worst(false, true, compaction_strategy); return true; } return false; diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index 8b122286211..558cc275f38 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -10,6 +10,7 @@ namespace vespalib::datastore { +class CompactionSpec; class CompactionStrategy; class DataStoreBase; @@ -30,6 +31,7 @@ public: using Index = enumstore::Index; using InternalIndex = enumstore::InternalIndex; using IndexVector = enumstore::IndexVector; + using CompactionSpec = vespalib::datastore::CompactionSpec; using CompactionStrategy = vespalib::datastore::CompactionStrategy; using EnumHandle = enumstore::EnumHandle; using EnumVector = enumstore::EnumVector; @@ -53,7 +55,7 @@ public: virtual vespalib::MemoryUsage get_dictionary_memory_usage() const = 0; virtual vespalib::MemoryUsage update_stat() = 0; virtual std::unique_ptr<EnumIndexRemapper> consider_compact_values(const CompactionStrategy& compaction_strategy) = 0; - virtual std::unique_ptr<EnumIndexRemapper> compact_worst_values(bool compact_memory, bool compact_address_space) = 0; + virtual std::unique_ptr<EnumIndexRemapper> compact_worst_values(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) = 0; virtual bool consider_compact_dictionary(const CompactionStrategy& compaction_strategy) = 0; virtual uint64_t get_compaction_count() const = 0; // Should only be used by unit tests. diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h index 9720e88543d..81abaa05a45 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h @@ -44,7 +44,7 @@ public: void doneLoadFromMultiValue() { _store.setInitializing(false); } - void compactWorst(bool compactMemory, bool compactAddressSpace) override; + void compactWorst(CompactionSpec compactionSpec, const CompactionStrategy& compaction_strategy) override; vespalib::AddressSpace getAddressSpaceUsage() const override; vespalib::MemoryUsage getArrayStoreMemoryUsage() const override; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp index 25065a200e9..fb81a60cb13 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp @@ -53,9 +53,9 @@ MultiValueMapping<EntryT,RefT>::replace(uint32_t docId, ConstArrayRef values) template <typename EntryT, typename RefT> void -MultiValueMapping<EntryT,RefT>::compactWorst(bool compactMemory, bool compactAddressSpace) +MultiValueMapping<EntryT,RefT>::compactWorst(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - vespalib::datastore::ICompactionContext::UP compactionContext(_store.compactWorst(compactMemory, compactAddressSpace)); + vespalib::datastore::ICompactionContext::UP compactionContext(_store.compactWorst(compaction_spec, compaction_strategy)); if (compactionContext) { compactionContext->compact(vespalib::ArrayRef<EntryRef>(&_indices[0], _indices.size())); } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp index 5e9c5758865..b44b2236f8f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "multi_value_mapping_base.h" +#include <vespa/vespalib/datastore/compaction_spec.h> #include <vespa/vespalib/datastore/compaction_strategy.h> #include <cassert> @@ -86,7 +87,8 @@ MultiValueMappingBase::considerCompact(const CompactionStrategy &compactionStrat bool compactMemory = compactionStrategy.should_compact_memory(usedBytes, deadBytes); bool compactAddressSpace = compactionStrategy.should_compact_address_space(usedArrays, deadArrays); if (compactMemory || compactAddressSpace) { - compactWorst(compactMemory, compactAddressSpace); + CompactionSpec compaction_spec(compactMemory, compactAddressSpace); + compactWorst(compaction_spec, compactionStrategy); return true; } return false; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h index 64c602e3c29..0034878fea6 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h @@ -7,7 +7,10 @@ #include <vespa/vespalib/util/rcuvector.h> #include <functional> -namespace vespalib::datastore { class CompactionStrategy; } +namespace vespalib::datastore { +class CompactionSpec; +class CompactionStrategy; +} namespace search::attribute { @@ -17,6 +20,7 @@ namespace search::attribute { class MultiValueMappingBase { public: + using CompactionSpec = vespalib::datastore::CompactionSpec; using CompactionStrategy = vespalib::datastore::CompactionStrategy; using EntryRef = vespalib::datastore::EntryRef; using RefVector = vespalib::RcuVectorBase<EntryRef>; @@ -52,7 +56,7 @@ public: uint32_t getNumKeys() const { return _indices.size(); } uint32_t getCapacityKeys() const { return _indices.capacity(); } - virtual void compactWorst(bool compatMemory, bool compactAddressSpace) = 0; + virtual void compactWorst(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) = 0; bool considerCompact(const CompactionStrategy &compactionStrategy); }; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 8ed8a0cfbee..8c4b8a66a7d 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreerootbase.cpp> #include <vespa/vespalib/datastore/datastore.hpp> +#include <vespa/vespalib/datastore/compaction_spec.h> #include <vespa/vespalib/datastore/entry_ref_filter.h> #include <vespa/vespalib/datastore/buffer_type.hpp> @@ -724,9 +725,9 @@ PostingStore<DataT>::move(std::vector<EntryRef>& refs) template <typename DataT> void -PostingStore<DataT>::compact_worst_btree_nodes() +PostingStore<DataT>::compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy) { - auto to_hold = this->start_compact_worst_btree_nodes(); + auto to_hold = this->start_compact_worst_btree_nodes(compaction_strategy); EntryRefFilter filter(RefType::numBuffers(), RefType::offset_bits); // Only look at buffers containing bitvectors and btree roots filter.add_buffers(this->_treeType.get_active_buffers()); @@ -738,9 +739,10 @@ PostingStore<DataT>::compact_worst_btree_nodes() template <typename DataT> void -PostingStore<DataT>::compact_worst_buffers() +PostingStore<DataT>::compact_worst_buffers(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - auto to_hold = this->start_compact_worst_buffers(); + + auto to_hold = this->start_compact_worst_buffers(compaction_spec, compaction_strategy); bool compact_btree_roots = false; EntryRefFilter filter(RefType::numBuffers(), RefType::offset_bits); filter.add_buffers(to_hold); @@ -769,7 +771,7 @@ PostingStore<DataT>::consider_compact_worst_btree_nodes(const CompactionStrategy return false; } if (compaction_strategy.should_compact_memory(_cached_allocator_memory_usage.usedBytes(), _cached_allocator_memory_usage.deadBytes())) { - compact_worst_btree_nodes(); + compact_worst_btree_nodes(compaction_strategy); return true; } return false; @@ -783,7 +785,8 @@ PostingStore<DataT>::consider_compact_worst_buffers(const CompactionStrategy& co return false; } if (compaction_strategy.should_compact_memory(_cached_store_memory_usage.usedBytes(), _cached_store_memory_usage.deadBytes())) { - compact_worst_buffers(); + CompactionSpec compaction_spec(true, false); + compact_worst_buffers(compaction_spec, compaction_strategy); return true; } return false; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h index e5506babb7f..58097194f50 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.h +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h @@ -77,6 +77,7 @@ public: typedef typename Parent::AggregatedType AggregatedType; typedef typename Parent::BTreeTypeRefPair BTreeTypeRefPair; typedef typename Parent::Builder Builder; + using CompactionSpec = vespalib::datastore::CompactionSpec; using CompactionStrategy = vespalib::datastore::CompactionStrategy; typedef vespalib::datastore::EntryRef EntryRef; typedef std::less<uint32_t> CompareT; @@ -191,8 +192,8 @@ public: void move_btree_nodes(const std::vector<EntryRef> &refs); void move(std::vector<EntryRef>& refs); - void compact_worst_btree_nodes(); - void compact_worst_buffers(); + void compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy); + void compact_worst_buffers(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy); bool consider_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy); bool consider_compact_worst_buffers(const CompactionStrategy& compaction_strategy); private: diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index eb822313d61..0a84da7c769 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -24,6 +24,7 @@ namespace search::attribute { using document::DocumentId; using document::GlobalId; using document::IdParseException; +using vespalib::datastore::CompactionSpec; namespace { @@ -295,16 +296,17 @@ ReferenceAttribute::consider_compact_values(const CompactionStrategy &compaction size_t dead_bytes = _cached_unique_store_values_memory_usage.deadBytes(); bool compact_memory = compactionStrategy.should_compact_memory(used_bytes, dead_bytes); if (compact_memory) { - compact_worst_values(); + compact_worst_values(compactionStrategy); return true; } return false; } void -ReferenceAttribute::compact_worst_values() +ReferenceAttribute::compact_worst_values(const CompactionStrategy& compaction_strategy) { - auto remapper(_store.compact_worst(true, true)); + CompactionSpec compaction_spec(true, true); + auto remapper(_store.compact_worst(compaction_spec, compaction_strategy)); if (remapper) { remapper->remap(vespalib::ArrayRef<EntryRef>(&_indices[0], _indices.size())); remapper->done(); @@ -321,7 +323,7 @@ ReferenceAttribute::consider_compact_dictionary(const CompactionStrategy &compac if (compaction_strategy.should_compact_memory(_cached_unique_store_dictionary_memory_usage.usedBytes(), _cached_unique_store_dictionary_memory_usage.deadBytes())) { - dictionary.compact_worst(true, true); + dictionary.compact_worst(true, true, compaction_strategy); return true; } return false; diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h index 23e75f973e5..237a0f1ddd7 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h @@ -58,7 +58,7 @@ private: uint64_t getUniqueValueCount() const override; bool consider_compact_values(const CompactionStrategy &compactionStrategy); - void compact_worst_values(); + void compact_worst_values(const CompactionStrategy& compaction_strategy); bool consider_compact_dictionary(const CompactionStrategy& compaction_strategy); IndicesCopyVector getIndicesCopy(uint32_t size) const; void removeReverseMapping(EntryRef oldRef, uint32_t lid); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 5e3a3856a4f..2c01472a067 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -532,18 +532,18 @@ HnswIndex::trim_hold_lists(generation_t first_used_gen) } void -HnswIndex::compact_level_arrays(bool compact_memory, bool compact_address_space) +HnswIndex::compact_level_arrays(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - auto context = _graph.nodes.compactWorst(compact_memory, compact_address_space); + auto context = _graph.nodes.compactWorst(compaction_spec, compaction_strategy); uint32_t doc_id_limit = _graph.node_refs.size(); vespalib::ArrayRef<AtomicEntryRef> refs(&_graph.node_refs[0], doc_id_limit); context->compact(refs); } void -HnswIndex::compact_link_arrays(bool compact_memory, bool compact_address_space) +HnswIndex::compact_link_arrays(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - auto context = _graph.links.compactWorst(compact_memory, compact_address_space); + auto context = _graph.links.compactWorst(compaction_spec, compaction_strategy); uint32_t doc_id_limit = _graph.node_refs.size(); for (uint32_t doc_id = 1; doc_id < doc_id_limit; ++doc_id) { EntryRef level_ref = _graph.node_refs[doc_id].load_relaxed(); @@ -557,7 +557,7 @@ HnswIndex::compact_link_arrays(bool compact_memory, bool compact_address_space) namespace { bool -consider_compact_arrays(const CompactionStrategy& compaction_strategy, vespalib::MemoryUsage& memory_usage, vespalib::AddressSpace& address_space_usage, std::function<void(bool,bool)> compact_arrays) +consider_compact_arrays(const CompactionStrategy& compaction_strategy, vespalib::MemoryUsage& memory_usage, vespalib::AddressSpace& address_space_usage, std::function<void(vespalib::datastore::CompactionSpec, const CompactionStrategy&)> compact_arrays) { size_t used_bytes = memory_usage.usedBytes(); size_t dead_bytes = memory_usage.deadBytes(); @@ -566,7 +566,8 @@ consider_compact_arrays(const CompactionStrategy& compaction_strategy, vespalib: size_t dead_address_space = address_space_usage.dead(); bool compact_address_space = compaction_strategy.should_compact_address_space(used_address_space, dead_address_space); if (compact_memory || compact_address_space) { - compact_arrays(compact_memory, compact_address_space); + vespalib::datastore::CompactionSpec compaction_spec(compact_memory, compact_address_space); + compact_arrays(compaction_spec, compaction_strategy); return true; } return false; @@ -578,16 +579,16 @@ bool HnswIndex::consider_compact_level_arrays(const CompactionStrategy& compaction_strategy) { return consider_compact_arrays(compaction_strategy, _cached_level_arrays_memory_usage, _cached_level_arrays_address_space_usage, - [this](bool compact_memory, bool compact_address_space) - { compact_level_arrays(compact_memory, compact_address_space); }); + [this](CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy_fwd) + { compact_level_arrays(compaction_spec, compaction_strategy_fwd); }); } bool HnswIndex::consider_compact_link_arrays(const CompactionStrategy& compaction_strategy) { return consider_compact_arrays(compaction_strategy, _cached_link_arrays_memory_usage, _cached_link_arrays_address_space_usage, - [this](bool compact_memory, bool compact_address_space) - { compact_link_arrays(compact_memory, compact_address_space); }); + [this](CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy_fwd) + { compact_link_arrays(compaction_spec, compaction_strategy_fwd); }); } bool diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index d8f3c4c97fa..5b5f9382517 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -171,8 +171,8 @@ public: void remove_document(uint32_t docid) override; void transfer_hold_lists(generation_t current_gen) override; void trim_hold_lists(generation_t first_used_gen) override; - void compact_level_arrays(bool compact_memory, bool compact_addreess_space); - void compact_link_arrays(bool compact_memory, bool compact_address_space); + void compact_level_arrays(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy); + void compact_link_arrays(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy); bool consider_compact_level_arrays(const CompactionStrategy& compaction_strategy); bool consider_compact_link_arrays(const CompactionStrategy& compaction_strategy); bool consider_compact(const CompactionStrategy& compaction_strategy) override; diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index ad83ab2b8ef..c1fa4da05d1 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -12,7 +12,10 @@ class FastOS_FileInterface; -namespace vespalib::datastore { class CompactionStrategy; } +namespace vespalib::datastore { +class CompactionSpec; +class CompactionStrategy; +} namespace vespalib::slime { struct Inserter; } namespace search::fileutil { class LoadedBuffer; } @@ -32,6 +35,7 @@ class NearestNeighborIndexSaver; */ class NearestNeighborIndex { public: + using CompactionSpec = vespalib::datastore::CompactionSpec; using CompactionStrategy = vespalib::datastore::CompactionStrategy; using generation_t = vespalib::GenerationHandler::generation_t; struct Neighbor { diff --git a/vespalib/src/tests/btree/btree_store/btree_store_test.cpp b/vespalib/src/tests/btree/btree_store/btree_store_test.cpp index 77cb8e519e4..974aafb392a 100644 --- a/vespalib/src/tests/btree/btree_store/btree_store_test.cpp +++ b/vespalib/src/tests/btree/btree_store/btree_store_test.cpp @@ -5,9 +5,12 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/gtest/gtest.h> using vespalib::GenerationHandler; +using vespalib::datastore::CompactionSpec; +using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; namespace vespalib::btree { @@ -131,7 +134,9 @@ BTreeStoreTest::test_compact_sequence(uint32_t sequence_length) move_refs.reserve(refs.size()); auto usage_before = store.getMemoryUsage(); for (uint32_t pass = 0; pass < 15; ++pass) { - auto to_hold = store.start_compact_worst_buffers(); + CompactionSpec compaction_spec(true, false); + CompactionStrategy compaction_strategy; + auto to_hold = store.start_compact_worst_buffers(compaction_spec, compaction_strategy); std::vector<bool> filter(TreeStore::RefType::numBuffers()); for (auto buffer_id : to_hold) { filter[buffer_id] = true; @@ -168,7 +173,8 @@ TEST_F(BTreeStoreTest, require_that_nodes_for_multiple_btrees_are_compacted) inc_generation(); auto usage_before = store.getMemoryUsage(); for (uint32_t pass = 0; pass < 15; ++pass) { - auto to_hold = store.start_compact_worst_btree_nodes(); + CompactionStrategy compaction_strategy; + auto to_hold = store.start_compact_worst_btree_nodes(compaction_strategy); store.move_btree_nodes(refs); store.finish_compact_worst_btree_nodes(to_hold); inc_generation(); diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index 4af0b9672f2..bd4f4f8ee08 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -17,6 +17,7 @@ #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/test/btree/btree_printer.h> #include <vespa/vespalib/gtest/gtest.h> @@ -24,6 +25,7 @@ LOG_SETUP("btree_test"); using vespalib::GenerationHandler; +using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; namespace vespalib::btree { @@ -1599,8 +1601,9 @@ TEST_F(BTreeTest, require_that_compaction_works) auto memory_usage_before = t.getAllocator().getMemoryUsage(); t.foreach_key([&before_list](int key) { before_list.emplace_back(key); }); make_iterators(t, before_list, before_iterators); + CompactionStrategy compaction_strategy; for (int i = 0; i < 15; ++i) { - t.compact_worst(); + t.compact_worst(compaction_strategy); } inc_generation(g, t); auto memory_usage_after = t.getAllocator().getMemoryUsage(); diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index dbd6d41f5e6..c58e357a9a1 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -3,6 +3,8 @@ #include <vespa/vespalib/test/datastore/buffer_stats.h> #include <vespa/vespalib/test/datastore/memstats.h> #include <vespa/vespalib/datastore/array_store.hpp> +#include <vespa/vespalib/datastore/compaction_spec.h> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -124,7 +126,9 @@ struct Fixture } template <typename TestedRefType> void compactWorst(bool compactMemory, bool compactAddressSpace) { - ICompactionContext::UP ctx = store.compactWorst(compactMemory, compactAddressSpace); + CompactionSpec compaction_spec(compactMemory, compactAddressSpace); + CompactionStrategy compaction_strategy; + ICompactionContext::UP ctx = store.compactWorst(compaction_spec, compaction_strategy); std::vector<TestedRefType> refs; for (auto itr = refStore.begin(); itr != refStore.end(); ++itr) { refs.emplace_back(itr->first); diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index ccb18f13871..917c91f2dff 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -1,4 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/datastore/compaction_spec.h> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/datastore/unique_store.hpp> #include <vespa/vespalib/datastore/unique_store_remapper.h> #include <vespa/vespalib/datastore/unique_store_string_allocator.hpp> @@ -111,7 +113,9 @@ struct TestBase : public ::testing::Test { store.trimHoldLists(generation); } void compactWorst() { - auto remapper = store.compact_worst(true, true); + CompactionSpec compaction_spec(true, true); + CompactionStrategy compaction_strategy; + auto remapper = store.compact_worst(compaction_spec, compaction_strategy); std::vector<EntryRef> refs; for (const auto &elem : refStore) { refs.push_back(elem.first); diff --git a/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp b/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp index 8d82c10d340..4a8b7eafe6a 100644 --- a/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp +++ b/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/datastore/unique_store.hpp> #include <vespa/vespalib/datastore/unique_store_dictionary.hpp> #include <vespa/vespalib/datastore/sharded_hash_map.h> @@ -137,8 +138,9 @@ TYPED_TEST(UniqueStoreDictionaryTest, compaction_works) this->inc_generation(); auto btree_memory_usage_before = this->dict.get_btree_memory_usage(); auto hash_memory_usage_before = this->dict.get_hash_memory_usage(); + CompactionStrategy compaction_strategy; for (uint32_t i = 0; i < 15; ++i) { - this->dict.compact_worst(true, true); + this->dict.compact_worst(true, true, compaction_strategy); } this->inc_generation(); auto btree_memory_usage_after = this->dict.get_btree_memory_usage(); diff --git a/vespalib/src/vespa/vespalib/btree/btree.h b/vespalib/src/vespa/vespalib/btree/btree.h index 2b03e70fbdf..f87d5751743 100644 --- a/vespalib/src/vespa/vespalib/btree/btree.h +++ b/vespalib/src/vespa/vespalib/btree/btree.h @@ -6,6 +6,8 @@ #include "noaggrcalc.h" #include <vespa/vespalib/util/generationhandler.h> +namespace vespalib::datastore { class CompactionStrategy; } + namespace vespalib::btree { /** @@ -149,7 +151,7 @@ public: _tree.thaw(itr); } - void compact_worst(); + void compact_worst(const datastore::CompactionStrategy& compaction_strategy); template <typename FunctionType> void diff --git a/vespalib/src/vespa/vespalib/btree/btree.hpp b/vespalib/src/vespa/vespalib/btree/btree.hpp index c4a588bc63e..473d1f4735e 100644 --- a/vespalib/src/vespa/vespalib/btree/btree.hpp +++ b/vespalib/src/vespa/vespalib/btree/btree.hpp @@ -26,9 +26,9 @@ BTree<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>::~BTree() template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, class AggrCalcT> void -BTree<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>::compact_worst() +BTree<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>::compact_worst(const datastore::CompactionStrategy& compaction_strategy) { - auto to_hold = _alloc.start_compact_worst(); + auto to_hold = _alloc.start_compact_worst(compaction_strategy); _tree.move_nodes(_alloc); _alloc.finishCompact(to_hold); } diff --git a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h index 93615ddef82..27e73b3a2b6 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h @@ -29,6 +29,7 @@ public: using BTreeRootBaseType = BTreeRootBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>; using generation_t = vespalib::GenerationHandler::generation_t; using NodeStore = BTreeNodeStore<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>; + using CompactionStrategy = datastore::CompactionStrategy; using EntryRef = datastore::EntryRef; using DataStoreBase = datastore::DataStoreBase; @@ -165,7 +166,7 @@ public: bool getCompacting(EntryRef ref) const { return _nodeStore.getCompacting(ref); } std::vector<uint32_t> startCompact() { return _nodeStore.startCompact(); } - std::vector<uint32_t> start_compact_worst() { return _nodeStore.start_compact_worst(); } + std::vector<uint32_t> start_compact_worst(const CompactionStrategy& compaction_strategy) { return _nodeStore.start_compact_worst(compaction_strategy); } void finishCompact(const std::vector<uint32_t> &toHold) { return _nodeStore.finishCompact(toHold); diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.h b/vespalib/src/vespa/vespalib/btree/btreenodestore.h index 70a9ba6c73c..444bf641899 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.h @@ -56,6 +56,7 @@ public: typedef typename LeafNodeType::RefPair LeafNodeTypeRefPair; typedef vespalib::GenerationHandler::generation_t generation_t; using EntryRef = datastore::EntryRef; + using CompactionStrategy = datastore::CompactionStrategy; enum NodeTypes { @@ -159,7 +160,7 @@ public: std::vector<uint32_t> startCompact(); - std::vector<uint32_t> start_compact_worst(); + std::vector<uint32_t> start_compact_worst(const CompactionStrategy& compaction_strategy); void finishCompact(const std::vector<uint32_t> &toHold); diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp index ff4fa59cd74..91953507eb0 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp @@ -3,6 +3,7 @@ #pragma once #include "btreenodestore.h" +#include <vespa/vespalib/datastore/compaction_spec.h> #include <vespa/vespalib/datastore/datastore.hpp> namespace vespalib::btree { @@ -71,9 +72,9 @@ template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS> std::vector<uint32_t> BTreeNodeStore<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: -start_compact_worst() +start_compact_worst(const CompactionStrategy &compaction_strategy) { - return _store.startCompactWorstBuffers(true, false); + return _store.startCompactWorstBuffers(datastore::CompactionSpec(true, false), compaction_strategy); } template <typename KeyT, typename DataT, typename AggrT, diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.h b/vespalib/src/vespa/vespalib/btree/btreestore.h index b4238757e46..a79259c6e57 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreestore.h @@ -49,6 +49,8 @@ public: TraitsT::INTERNAL_SLOTS, TraitsT::LEAF_SLOTS, AggrCalcT> Builder; + using CompactionSpec = datastore::CompactionSpec; + using CompactionStrategy = datastore::CompactionStrategy; using EntryRef = datastore::EntryRef; template <typename EntryType> using BufferType = datastore::BufferType<EntryType>; @@ -392,11 +394,11 @@ public: void foreach_frozen(EntryRef ref, FunctionType func) const; - std::vector<uint32_t> start_compact_worst_btree_nodes(); + std::vector<uint32_t> start_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy); void finish_compact_worst_btree_nodes(const std::vector<uint32_t>& to_hold); void move_btree_nodes(const std::vector<EntryRef>& refs); - std::vector<uint32_t> start_compact_worst_buffers(); + std::vector<uint32_t> start_compact_worst_buffers(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy); void move(std::vector<EntryRef>& refs); private: diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.hpp b/vespalib/src/vespa/vespalib/btree/btreestore.hpp index 795e526f927..c0985ff8f94 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreestore.hpp @@ -5,6 +5,7 @@ #include "btreestore.h" #include "btreebuilder.h" #include "btreebuilder.hpp" +#include <vespa/vespalib/datastore/compaction_spec.h> #include <vespa/vespalib/datastore/datastore.hpp> #include <vespa/vespalib/util/optimized.h> @@ -972,10 +973,10 @@ template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, typename AggrCalcT> std::vector<uint32_t> BTreeStore<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>:: -start_compact_worst_btree_nodes() +start_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy) { _builder.clear(); - return _allocator.start_compact_worst(); + return _allocator.start_compact_worst(compaction_strategy); } template <typename KeyT, typename DataT, typename AggrT, typename CompareT, @@ -1007,10 +1008,10 @@ template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, typename AggrCalcT> std::vector<uint32_t> BTreeStore<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>:: -start_compact_worst_buffers() +start_compact_worst_buffers(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { freeze(); - return _store.startCompactWorstBuffers(true, false); + return _store.startCompactWorstBuffers(compaction_spec, compaction_strategy); } template <typename KeyT, typename DataT, typename AggrT, typename CompareT, diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index 3ba0caae5b9..d9b62c310b5 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -96,7 +96,7 @@ public: } void remove(EntryRef ref); - ICompactionContext::UP compactWorst(bool compactMemory, bool compactAddressSpace); + ICompactionContext::UP compactWorst(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy); vespalib::MemoryUsage getMemoryUsage() const { return _store.getMemoryUsage(); } /** diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp index 9317fa557c0..bbbd52c354d 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp @@ -3,6 +3,7 @@ #pragma once #include "array_store.h" +#include "compaction_spec.h" #include "entry_ref_filter.h" #include "datastore.hpp" #include <atomic> @@ -169,9 +170,9 @@ public: template <typename EntryT, typename RefT> ICompactionContext::UP -ArrayStore<EntryT, RefT>::compactWorst(bool compactMemory, bool compactAddressSpace) +ArrayStore<EntryT, RefT>::compactWorst(CompactionSpec compaction_spec, const CompactionStrategy &compaction_strategy) { - std::vector<uint32_t> bufferIdsToCompact = _store.startCompactWorstBuffers(compactMemory, compactAddressSpace); + std::vector<uint32_t> bufferIdsToCompact = _store.startCompactWorstBuffers(compaction_spec, compaction_strategy); return std::make_unique<arraystore::CompactionContext<EntryT, RefT>> (_store, *this, std::move(bufferIdsToCompact)); } diff --git a/vespalib/src/vespa/vespalib/datastore/compaction_spec.h b/vespalib/src/vespa/vespalib/datastore/compaction_spec.h new file mode 100644 index 00000000000..49e2f5057cb --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/compaction_spec.h @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib::datastore { + +/* + * Class describing how to compact a compactable data structure. + * + * memory - to reduce amount of "dead" memory + * address_space - to avoid running out of free buffers in data store + * (i.e. move data from small buffers to larger buffers) + */ +class CompactionSpec +{ + bool _compact_memory; + bool _compact_address_space; +public: + CompactionSpec(bool compact_memory, bool compact_address_space) + : _compact_memory(compact_memory), + _compact_address_space(compact_address_space) + { + } + bool get_compact_memory() const noexcept { return _compact_memory; } + bool get_compact_address_space() const noexcept { return _compact_address_space; } +}; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp index b5cab50bc33..367eb328ab2 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "datastore.h" +#include "compaction_spec.h" #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <limits> @@ -526,8 +527,9 @@ DataStoreBase::markCompacting(uint32_t bufferId) } std::vector<uint32_t> -DataStoreBase::startCompactWorstBuffers(bool compactMemory, bool compactAddressSpace) +DataStoreBase::startCompactWorstBuffers(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { + (void) compaction_strategy; constexpr uint32_t noBufferId = std::numeric_limits<uint32_t>::max(); uint32_t worstMemoryBufferId = noBufferId; uint32_t worstAddressSpaceBufferId = noBufferId; @@ -540,11 +542,11 @@ DataStoreBase::startCompactWorstBuffers(bool compactMemory, bool compactAddressS uint32_t arraySize = typeHandler->getArraySize(); uint32_t reservedElements = typeHandler->getReservedElements(bufferId); size_t deadElems = state.getDeadElems() - reservedElements; - if (compactMemory && deadElems > worstDeadElems) { + if (compaction_spec.get_compact_memory() && deadElems > worstDeadElems) { worstMemoryBufferId = bufferId; worstDeadElems = deadElems; } - if (compactAddressSpace) { + if (compaction_spec.get_compact_address_space()) { size_t deadArrays = deadElems / arraySize; if (deadArrays > worstDeadArrays) { worstAddressSpaceBufferId = bufferId; diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.h b/vespalib/src/vespa/vespalib/datastore/datastorebase.h index 6903ae12c9c..e98d9531806 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.h +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.h @@ -12,6 +12,9 @@ namespace vespalib::datastore { +class CompactionSpec; +class CompactionStrategy; + /** * Abstract class used to store data of potential different types in underlying memory buffers. * @@ -368,7 +371,7 @@ public: } uint32_t startCompactWorstBuffer(uint32_t typeId); - std::vector<uint32_t> startCompactWorstBuffers(bool compactMemory, bool compactAddressSpace); + std::vector<uint32_t> startCompactWorstBuffers(CompactionSpec compaction_spec, const CompactionStrategy &compaction_strategy); uint64_t get_compaction_count() const { return _compaction_count.load(std::memory_order_relaxed); } void inc_compaction_count() const { ++_compaction_count; } bool has_held_buffers() const noexcept { return _hold_buffer_count != 0u; } diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h index cf848167070..4fd3bcad5e5 100644 --- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h @@ -10,6 +10,7 @@ namespace vespalib::datastore { +class CompactionStrategy; class EntryComparator; class EntryRefFilter; struct ICompactable; @@ -41,7 +42,7 @@ public: virtual vespalib::MemoryUsage get_btree_memory_usage() const = 0; virtual vespalib::MemoryUsage get_hash_memory_usage() const = 0; virtual bool has_held_buffers() const = 0; - virtual void compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary) = 0; + virtual void compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary, const CompactionStrategy& compaction_strategy) = 0; }; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.h b/vespalib/src/vespa/vespalib/datastore/unique_store.h index 38643d84be0..b50f753ed03 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.h @@ -55,7 +55,7 @@ public: EntryRef find(EntryConstRefType value); EntryConstRefType get(EntryRef ref) const { return _allocator.get(ref); } void remove(EntryRef ref); - std::unique_ptr<Remapper> compact_worst(bool compact_memory, bool compact_address_space); + std::unique_ptr<Remapper> compact_worst(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy); vespalib::MemoryUsage getMemoryUsage() const; vespalib::MemoryUsage get_values_memory_usage() const { return _store.getMemoryUsage(); } vespalib::MemoryUsage get_dictionary_memory_usage() const { return _dict->get_memory_usage(); } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index b02a2e52185..2c06788ded8 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -157,9 +157,9 @@ public: template <typename EntryT, typename RefT, typename Compare, typename Allocator> std::unique_ptr<typename UniqueStore<EntryT, RefT, Compare, Allocator>::Remapper> -UniqueStore<EntryT, RefT, Compare, Allocator>::compact_worst(bool compact_memory, bool compact_address_space) +UniqueStore<EntryT, RefT, Compare, Allocator>::compact_worst(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - std::vector<uint32_t> bufferIdsToCompact = _store.startCompactWorstBuffers(compact_memory, compact_address_space); + std::vector<uint32_t> bufferIdsToCompact = _store.startCompactWorstBuffers(compaction_spec, compaction_strategy); if (bufferIdsToCompact.empty()) { return std::unique_ptr<Remapper>(); } else { diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h index 54d541853c7..d64588e3242 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h @@ -91,7 +91,7 @@ public: vespalib::MemoryUsage get_btree_memory_usage() const override; vespalib::MemoryUsage get_hash_memory_usage() const override; bool has_held_buffers() const override; - void compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary) override; + void compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary, const CompactionStrategy& compaction_strategy) override; }; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp index 13ae0a317e0..4375b38cf7c 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -339,11 +339,11 @@ UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::has_held_buff template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary) +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::compact_worst(bool compact_btree_dictionary, bool compact_hash_dictionary, const CompactionStrategy& compaction_strategy) { if constexpr (has_btree_dictionary) { if (compact_btree_dictionary) { - this->_btree_dict.compact_worst(); + this->_btree_dict.compact_worst(compaction_strategy); } } else { (void) compact_btree_dictionary; |