diff options
author | Tor Egge <Tor.Egge@online.no> | 2021-12-08 13:07:05 +0100 |
---|---|---|
committer | Tor Egge <Tor.Egge@online.no> | 2021-12-08 13:07:05 +0100 |
commit | 7553a7267623edf2c45dd5a0fd1cea883ab691f7 (patch) | |
tree | a928677c7b0ac2d4b60ed0f5b33c0ba815210ac2 /searchlib | |
parent | 2dd0a7ad258fa182ed3ac8a199751dd60f73b0f7 (diff) |
Move CompactionStrategy from searchcommon to vespalib.
Diffstat (limited to 'searchlib')
16 files changed, 37 insertions, 32 deletions
diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp index 801aa9341fb..fbec89d27eb 100644 --- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp @@ -16,6 +16,7 @@ using search::attribute::Config; using search::attribute::BasicType; using search::attribute::CollectionType; using vespalib::AddressSpace; +using vespalib::datastore::CompactionStrategy; using AttributePtr = AttributeVector::SP; using AttributeStatus = search::attribute::Status; @@ -237,7 +238,7 @@ TEST_F("Compaction limits address space usage (dead) when free lists are NOT use { populate_and_hammer(f, true); AddressSpace afterSpace = f.getMultiValueAddressSpaceUsage("after"); - EXPECT_GREATER(search::CompactionStrategy::DEAD_ADDRESS_SPACE_SLACK, afterSpace.dead()); + EXPECT_GREATER(CompactionStrategy::DEAD_ADDRESS_SPACE_SLACK, afterSpace.dead()); } TEST_F("Compaction is not executed when free lists are used", @@ -266,7 +267,7 @@ TEST_F("Compaction is peformed when compaction strategy is changed to enable com f._v->commit(); // new commit might trigger further compaction after2 = f.getMultiValueAddressSpaceUsage("after2"); } - EXPECT_GREATER(search::CompactionStrategy::DEAD_ADDRESS_SPACE_SLACK, after2.dead()); + EXPECT_GREATER(CompactionStrategy::DEAD_ADDRESS_SPACE_SLACK, after2.dead()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp b/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp index 2c8fc2966b0..b30b3e4eb71 100644 --- a/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp @@ -175,7 +175,7 @@ void CompactionTest<VectorType>::test_enum_store_compaction() { constexpr uint32_t canary_stride = 256; - uint32_t dead_limit = search::CompactionStrategy::DEAD_BYTES_SLACK / 8; + uint32_t dead_limit = vespalib::datastore::CompactionStrategy::DEAD_BYTES_SLACK / 8; uint32_t doc_count = dead_limit * 3; if (_v->hasMultiValue() || std::is_same_v<VectorType,StringAttribute>) { doc_count /= 2; diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index 40ff25ff976..06ee0060c7a 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -7,6 +7,7 @@ LOG_SETUP("enumstore_test"); using Type = search::DictionaryConfig::Type; +using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; using vespalib::datastore::EntryRefFilter; using RefT = vespalib::datastore::EntryRefT<22>; @@ -878,7 +879,7 @@ void inc_generation(generation_t &gen, NumericEnumStore &store) TYPED_TEST(EnumStoreDictionaryTest, compact_worst_works) { - size_t entry_count = (search::CompactionStrategy::DEAD_BYTES_SLACK / 8) + 40; + size_t entry_count = (CompactionStrategy::DEAD_BYTES_SLACK / 8) + 40; auto updater = this->store.make_batch_updater(); for (int32_t i = 0; (size_t) i < entry_count; ++i) { auto idx = updater.insert(i); @@ -891,13 +892,13 @@ TYPED_TEST(EnumStoreDictionaryTest, compact_worst_works) inc_generation(gen, this->store); auto& dict = this->store.get_dictionary(); if (dict.get_has_btree_dictionary()) { - EXPECT_LT(search::CompactionStrategy::DEAD_BYTES_SLACK, dict.get_btree_memory_usage().deadBytes()); + EXPECT_LT(CompactionStrategy::DEAD_BYTES_SLACK, dict.get_btree_memory_usage().deadBytes()); } if (dict.get_has_hash_dictionary()) { - EXPECT_LT(search::CompactionStrategy::DEAD_BYTES_SLACK, dict.get_hash_memory_usage().deadBytes()); + EXPECT_LT(CompactionStrategy::DEAD_BYTES_SLACK, dict.get_hash_memory_usage().deadBytes()); } int compact_count = 0; - search::CompactionStrategy compaction_strategy; + CompactionStrategy compaction_strategy; for (uint32_t i = 0; i < 15; ++i) { this->store.update_stat(); if (this->store.consider_compact_dictionary(compaction_strategy)) { @@ -911,10 +912,10 @@ TYPED_TEST(EnumStoreDictionaryTest, compact_worst_works) EXPECT_LT((TypeParam::type == Type::BTREE_AND_HASH) ? 1 : 0, compact_count); EXPECT_GT(15, compact_count); if (dict.get_has_btree_dictionary()) { - EXPECT_GT(search::CompactionStrategy::DEAD_BYTES_SLACK, dict.get_btree_memory_usage().deadBytes()); + EXPECT_GT(CompactionStrategy::DEAD_BYTES_SLACK, dict.get_btree_memory_usage().deadBytes()); } if (dict.get_has_hash_dictionary()) { - EXPECT_GT(search::CompactionStrategy::DEAD_BYTES_SLACK, dict.get_hash_memory_usage().deadBytes()); + EXPECT_GT(CompactionStrategy::DEAD_BYTES_SLACK, dict.get_hash_memory_usage().deadBytes()); } std::vector<int32_t> exp_values; std::vector<int32_t> values; diff --git a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp index cd78332cacd..34b8603c63c 100644 --- a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp +++ b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp @@ -13,6 +13,7 @@ #include <ostream> using vespalib::GenerationHandler; +using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; namespace search::attribute { @@ -160,7 +161,7 @@ PostingStoreTest::test_compact_sequence(uint32_t sequence_length) EntryRef old_ref2 = get_posting_ref(2); auto usage_before = store.getMemoryUsage(); bool compaction_done = false; - search::CompactionStrategy compaction_strategy(0.05, 0.2); + CompactionStrategy compaction_strategy(0.05, 0.2); for (uint32_t pass = 0; pass < 45; ++pass) { store.update_stat(); auto guard = _gen_handler.takeGuard(); @@ -193,7 +194,7 @@ PostingStoreTest::test_compact_btree_nodes(uint32_t sequence_length) EntryRef old_ref2 = get_posting_ref(2); auto usage_before = store.getMemoryUsage(); bool compaction_done = false; - search::CompactionStrategy compaction_strategy(0.05, 0.2); + CompactionStrategy compaction_strategy(0.05, 0.2); for (uint32_t pass = 0; pass < 55; ++pass) { store.update_stat(); auto guard = _gen_handler.takeGuard(); diff --git a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp index c077ab83a6e..1a8eda40f52 100644 --- a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp +++ b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp @@ -176,7 +176,7 @@ struct ReferenceAttributeTest : public ::testing::Test { search::attribute::Status newStatus = oldStatus; uint64_t iter = 0; AttributeGuard guard(_attr); - uint64_t dropCount = search::CompactionStrategy::DEAD_BYTES_SLACK / sizeof(Reference); + uint64_t dropCount = vespalib::datastore::CompactionStrategy::DEAD_BYTES_SLACK / sizeof(Reference); for (; iter < iterLimit; ++iter) { clear(2); set(2, toGid(doc2)); diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 922b0d4fb3e..f47e392c047 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -38,7 +38,6 @@ using document::WrongTensorTypeException; using search::AddressSpaceUsage; using search::AttributeGuard; using search::AttributeVector; -using search::CompactionStrategy; using search::attribute::DistanceMetric; using search::attribute::HnswIndexParams; using search::queryeval::GlobalFilter; @@ -56,6 +55,7 @@ using search::tensor::NearestNeighborIndexLoader; using search::tensor::NearestNeighborIndexSaver; using search::tensor::PrepareResult; using search::tensor::TensorAttribute; +using vespalib::datastore::CompactionStrategy; using vespalib::eval::TensorSpec; using vespalib::eval::CellType; using vespalib::eval::ValueType; diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 7acd3cf8b57..c35aba92c56 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchcommon/common/compaction_strategy.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/tensor/distance_functions.h> #include <vespa/searchlib/tensor/doc_vector_access.h> #include <vespa/searchlib/tensor/hnsw_index.h> #include <vespa/searchlib/tensor/random_level_generator.h> #include <vespa/searchlib/tensor/inv_log_level_generator.h> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/data/slime/slime.h> @@ -21,7 +21,7 @@ using namespace search::tensor; using namespace vespalib::slime; using vespalib::Slime; using search::BitVector; -using search::CompactionStrategy; +using vespalib::datastore::CompactionStrategy; template <typename FloatType> class MyDocVectorAccess : public DocVectorAccess { diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index c202d780659..897a2153b2b 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -18,10 +18,11 @@ #include <vespa/vespalib/datastore/unique_store_string_allocator.hpp> #include <vespa/vespalib/util/array.hpp> #include <vespa/searchlib/util/bufferwriter.h> -#include <vespa/searchcommon/common/compaction_strategy.h> +#include <vespa/vespalib/datastore/compaction_strategy.h> namespace search { +using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryComparator; std::unique_ptr<vespalib::datastore::IUniqueStoreDictionary> diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index 1f3165828bc..8b122286211 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -10,6 +10,7 @@ namespace vespalib::datastore { +class CompactionStrategy; class DataStoreBase; template <typename> class UniqueStoreRemapper; @@ -19,7 +20,6 @@ template <typename> class UniqueStoreRemapper; namespace search { class BufferWriter; -class CompactionStrategy; class IEnumStoreDictionary; /** @@ -30,6 +30,7 @@ public: using Index = enumstore::Index; using InternalIndex = enumstore::InternalIndex; using IndexVector = enumstore::IndexVector; + using CompactionStrategy = vespalib::datastore::CompactionStrategy; using EnumHandle = enumstore::EnumHandle; using EnumVector = enumstore::EnumVector; using EnumIndexRemapper = vespalib::datastore::UniqueStoreRemapper<InternalIndex>; diff --git a/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h b/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h index 135870e29a5..20cec9a31c2 100644 --- a/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h +++ b/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h @@ -4,7 +4,7 @@ #include <vespa/searchcommon/attribute/iattributevector.h> -namespace search { class CompactionStrategy; } +namespace vespalib::datastore { class CompactionStrategy; } namespace vespalib { class MemoryUsage; } @@ -13,16 +13,9 @@ namespace search::attribute { class IPostingListAttributeBase { public: - virtual - ~IPostingListAttributeBase() - { - } - - virtual void - clearPostings(IAttributeVector::EnumHandle eidx, - uint32_t fromLid, - uint32_t toLid) = 0; - + using CompactionStrategy = vespalib::datastore::CompactionStrategy; + virtual ~IPostingListAttributeBase() = default; + virtual void clearPostings(IAttributeVector::EnumHandle eidx, uint32_t fromLid, uint32_t toLid) = 0; virtual void forwardedShrinkLidSpace(uint32_t newSize) = 0; virtual vespalib::MemoryUsage getMemoryUsage() const = 0; virtual bool consider_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy) = 0; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp index 2edc30cc2c4..5e9c5758865 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp @@ -1,11 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "multi_value_mapping_base.h" -#include <vespa/searchcommon/common/compaction_strategy.h> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <cassert> namespace search::attribute { +using vespalib::datastore::CompactionStrategy; + MultiValueMappingBase::MultiValueMappingBase(const vespalib::GrowStrategy &gs, vespalib::GenerationHolder &genHolder) : _indices(gs, genHolder), diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h index 952e9dbbe56..64c602e3c29 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h @@ -7,7 +7,7 @@ #include <vespa/vespalib/util/rcuvector.h> #include <functional> -namespace search { class CompactionStrategy; } +namespace vespalib::datastore { class CompactionStrategy; } namespace search::attribute { @@ -17,6 +17,7 @@ namespace search::attribute { class MultiValueMappingBase { public: + using CompactionStrategy = vespalib::datastore::CompactionStrategy; using EntryRef = vespalib::datastore::EntryRef; using RefVector = vespalib::RcuVectorBase<EntryRef>; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h index 2b119a55158..e5506babb7f 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.h +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h @@ -77,6 +77,7 @@ public: typedef typename Parent::AggregatedType AggregatedType; typedef typename Parent::BTreeTypeRefPair BTreeTypeRefPair; typedef typename Parent::Builder Builder; + using CompactionStrategy = vespalib::datastore::CompactionStrategy; typedef vespalib::datastore::EntryRef EntryRef; typedef std::less<uint32_t> CompareT; using Parent::applyNewArray; diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h index 4016230ef89..23e75f973e5 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h @@ -25,6 +25,7 @@ namespace search::attribute { class ReferenceAttribute : public NotImplementedAttribute { public: + using CompactionStrategy = vespalib::datastore::CompactionStrategy; using EntryRef = vespalib::datastore::EntryRef; using GlobalId = document::GlobalId; using ReferenceStore = vespalib::datastore::UniqueStore<Reference>; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 2889dc425db..5e3a3856a4f 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -8,7 +8,6 @@ #include "hnsw_index_saver.h" #include "random_level_generator.h" #include "reusable_set_visited_tracker.h" -#include <vespa/searchcommon/common/compaction_strategy.h> #include <vespa/searchlib/attribute/address_space_components.h> #include <vespa/searchlib/attribute/address_space_usage.h> #include <vespa/searchlib/util/fileutil.h> @@ -16,6 +15,7 @@ #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/datastore/array_store.hpp> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/util/memory_allocator.h> #include <vespa/vespalib/util/rcuvector.hpp> #include <vespa/vespalib/util/size_literals.h> @@ -30,6 +30,7 @@ namespace search::tensor { using search::AddressSpaceComponents; using search::StateExplorerUtils; +using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; namespace { diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index 411d09cd2d3..ad83ab2b8ef 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -12,6 +12,7 @@ class FastOS_FileInterface; +namespace vespalib::datastore { class CompactionStrategy; } namespace vespalib::slime { struct Inserter; } namespace search::fileutil { class LoadedBuffer; } @@ -19,7 +20,6 @@ namespace search::fileutil { class LoadedBuffer; } namespace search { class AddressSpaceUsage; class BitVector; -class CompactionStrategy; } namespace search::tensor { @@ -32,6 +32,7 @@ class NearestNeighborIndexSaver; */ class NearestNeighborIndex { public: + using CompactionStrategy = vespalib::datastore::CompactionStrategy; using generation_t = vespalib::GenerationHandler::generation_t; struct Neighbor { uint32_t docid; |