diff options
9 files changed, 120 insertions, 92 deletions
diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index d867ae9f211..3b3fdd9bc5c 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -1,13 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "enum_store_dictionary.h" -#include "enumstore.h" #include <vespa/vespalib/btree/btree.hpp> -#include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreenode.hpp> -#include <vespa/vespalib/btree/btreenodeallocator.hpp> -#include <vespa/vespalib/btree/btreeroot.hpp> -#include <vespa/vespalib/datastore/datastore.hpp> #include <vespa/vespalib/datastore/sharded_hash_map.h> #include <vespa/vespalib/datastore/unique_store_dictionary.hpp> #include <vespa/searchlib/util/bufferwriter.h> @@ -15,7 +10,6 @@ #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.enum_store_dictionary"); -using vespalib::datastore::EntryComparator; using vespalib::datastore::EntryRef; using vespalib::datastore::UniqueStoreAddResult; @@ -25,12 +19,8 @@ using vespalib::btree::BTreeNode; template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::remove_unused_values(const IndexSet& unused, - const vespalib::datastore::EntryComparator& cmp) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::remove_unused_values(const IndexList & unused,const EntryComparator& cmp) { - if (unused.empty()) { - return; - } for (const auto& ref : unused) { this->remove(cmp, ref); } @@ -48,9 +38,9 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::~EnumStoreDictionary() = template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const vespalib::datastore::EntryComparator& cmp) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const EntryComparator& cmp) { - IndexSet unused; + IndexList unused; // find unused enums if constexpr (has_btree_dictionary) { @@ -58,19 +48,26 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const _enumStore.free_value_if_unused(iter.getKey(), unused); } } else { - this->_hash_dict.foreach_key([this, &unused](EntryRef ref) { _enumStore.free_value_if_unused(ref, unused); }); + this->_hash_dict.foreach_key([this, &unused](EntryRef ref) { + _enumStore.free_value_if_unused(ref, unused); + }); } remove_unused_values(unused, cmp); } template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const IndexSet& to_remove, - const vespalib::datastore::EntryComparator& cmp) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const IndexList& to_remove, const EntryComparator& cmp) { - IndexSet unused; + IndexList unused; + + EntryRef prev; for (const auto& index : to_remove) { - _enumStore.free_value_if_unused(index, unused); + assert(prev <= index); + if (index != prev) { + _enumStore.free_value_if_unused(index, unused); + prev = index; + } } remove_unused_values(unused, cmp); } @@ -96,8 +93,7 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::remove(const EntryCompar template <typename BTreeDictionaryT, typename HashDictionaryT> bool -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_index(const vespalib::datastore::EntryComparator& cmp, - Index& idx) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_index(const EntryComparator& cmp, Index& idx) const { if constexpr (has_hash_dictionary) { auto find_result = this->_hash_dict.find(cmp, EntryRef()); @@ -118,8 +114,7 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_index(const vespali template <typename BTreeDictionaryT, typename HashDictionaryT> bool -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_frozen_index(const vespalib::datastore::EntryComparator& cmp, - Index& idx) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_frozen_index(const EntryComparator& cmp, Index& idx) const { if constexpr (has_hash_dictionary) { auto find_result = this->_hash_dict.find(cmp, EntryRef()); @@ -140,7 +135,7 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_frozen_index(const template <typename BTreeDictionaryT, typename HashDictionaryT> std::vector<IEnumStore::EnumHandle> -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_matching_enums(const EntryComparator& cmp) const { std::vector<IEnumStore::EnumHandle> result; if constexpr (has_btree_dictionary) { @@ -171,14 +166,14 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::get_frozen_root() const template <> std::pair<IEnumStore::Index, EntryRef> -EnumStoreDictionary<EnumTree>::find_posting_list(const vespalib::datastore::EntryComparator&, EntryRef) const +EnumStoreDictionary<EnumTree>::find_posting_list(const EntryComparator&, EntryRef) const { LOG_ABORT("should not be reached"); } template <typename BTreeDictionaryT, typename HashDictionaryT> std::pair<IEnumStore::Index, EntryRef> -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_posting_list(const EntryComparator& cmp, EntryRef root) const { if constexpr (has_hash_dictionary) { (void) root; @@ -199,7 +194,7 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_posting_list(const template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::collect_folded(Index idx, EntryRef, const std::function<void(vespalib::datastore::EntryRef)>& callback) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::collect_folded(Index idx, EntryRef, const std::function<void(EntryRef)>& callback) const { callback(idx); } @@ -244,14 +239,14 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::clear_all_posting_lists( template <> void -EnumStoreDictionary<EnumTree>::update_posting_list(Index, const vespalib::datastore::EntryComparator&, std::function<EntryRef(EntryRef)>) +EnumStoreDictionary<EnumTree>::update_posting_list(Index, const EntryComparator&, std::function<EntryRef(EntryRef)>) { LOG_ABORT("should not be reached"); } template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::update_posting_list(Index idx, const EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) { if constexpr (has_btree_dictionary) { auto& dict = this->_btree_dict; @@ -336,7 +331,7 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::get_posting_dictionary() return this->_btree_dict; } -EnumStoreFoldedDictionary::EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr<vespalib::datastore::EntryComparator> compare, std::unique_ptr<EntryComparator> folded_compare) +EnumStoreFoldedDictionary::EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr<EntryComparator> compare, std::unique_ptr<EntryComparator> folded_compare) : EnumStoreDictionary<EnumPostingTree>(enumStore, std::move(compare)), _folded_compare(std::move(folded_compare)) { @@ -389,7 +384,7 @@ EnumStoreFoldedDictionary::remove(const EntryComparator& comp, EntryRef ref) } void -EnumStoreFoldedDictionary::collect_folded(Index idx, EntryRef root, const std::function<void(vespalib::datastore::EntryRef)>& callback) const +EnumStoreFoldedDictionary::collect_folded(Index idx, EntryRef root, const std::function<void(EntryRef)>& callback) const { BTreeDictionaryType::ConstIterator itr(vespalib::btree::BTreeNode::Ref(), _btree_dict.getAllocator()); itr.lower_bound(root, idx, *_folded_compare); @@ -421,6 +416,7 @@ namespace vespalib::btree { using search::IEnumStore; using search::EnumTreeTraits; +using datastore::EntryComparatorWrapper; template class BTreeNodeT<IEnumStore::Index, EnumTreeTraits::INTERNAL_SLOTS>; @@ -456,19 +452,19 @@ class BTreeNodeStore<IEnumStore::Index, uint32_t, NoAggregated, template class BTreeRoot<IEnumStore::Index, BTreeNoLeafData, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRoot<IEnumStore::Index, uint32_t, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRootT<IEnumStore::Index, BTreeNoLeafData, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRootT<IEnumStore::Index, uint32_t, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRootBase<IEnumStore::Index, BTreeNoLeafData, NoAggregated, @@ -494,23 +490,23 @@ class BTreeIteratorBase<IEnumStore::Index, uint32_t, NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS, EnumTreeTraits::PATH_SIZE>; template class BTreeConstIterator<IEnumStore::Index, BTreeNoLeafData, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeConstIterator<IEnumStore::Index, uint32_t, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeIterator<IEnumStore::Index, BTreeNoLeafData, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeIterator<IEnumStore::Index, uint32_t, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTree<IEnumStore::Index, BTreeNoLeafData, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTree<IEnumStore::Index, uint32_t, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; } diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h index a39ff524618..3626fb098d2 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h @@ -18,9 +18,10 @@ protected: using EntryRef = IEnumStoreDictionary::EntryRef; using Index = IEnumStoreDictionary::Index; using BTreeDictionaryType = BTreeDictionaryT; + using EntryComparator = IEnumStoreDictionary::EntryComparator; private: using EnumVector = IEnumStoreDictionary::EnumVector; - using IndexSet = IEnumStoreDictionary::IndexSet; + using IndexList = IEnumStoreDictionary::IndexList; using IndexVector = IEnumStoreDictionary::IndexVector; using ParentUniqueStoreDictionary = vespalib::datastore::UniqueStoreDictionary<BTreeDictionaryT, IEnumStoreDictionary, HashDictionaryT>; using generation_t = IEnumStoreDictionary::generation_t; @@ -30,31 +31,28 @@ protected: private: IEnumStore& _enumStore; - void remove_unused_values(const IndexSet& unused, - const vespalib::datastore::EntryComparator& cmp); + void remove_unused_values(const IndexList& unused, const EntryComparator& cmp); public: - EnumStoreDictionary(IEnumStore& enumStore, std::unique_ptr<vespalib::datastore::EntryComparator> compare); + EnumStoreDictionary(IEnumStore& enumStore, std::unique_ptr<EntryComparator> compare); ~EnumStoreDictionary() override; - void free_unused_values(const vespalib::datastore::EntryComparator& cmp) override; + void free_unused_values(const EntryComparator& cmp) override; + void free_unused_values(const IndexList& to_remove, const EntryComparator& cmp) override; - void free_unused_values(const IndexSet& to_remove, - const vespalib::datastore::EntryComparator& cmp) override; - - void remove(const vespalib::datastore::EntryComparator& comp, vespalib::datastore::EntryRef ref) override; - bool find_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const override; - bool find_frozen_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const override; + void remove(const EntryComparator& comp, EntryRef ref) override; + bool find_index(const EntryComparator& cmp, Index& idx) const override; + bool find_frozen_index(const EntryComparator& cmp, Index& idx) const override; std::vector<attribute::IAttributeVector::EnumHandle> - find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const override; + find_matching_enums(const EntryComparator& cmp) const override; EntryRef get_frozen_root() const override; - std::pair<Index, EntryRef> find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const override; - void collect_folded(Index idx, EntryRef root, const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; + std::pair<Index, EntryRef> find_posting_list(const EntryComparator& cmp, EntryRef root) const override; + void collect_folded(Index idx, EntryRef root, const std::function<void(EntryRef)>& callback) const override; Index remap_index(Index idx) override; void clear_all_posting_lists(std::function<void(EntryRef)> clearer) override; - void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) override; + void update_posting_list(Index idx, const EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) override; bool normalize_posting_lists(std::function<EntryRef(EntryRef)> normalize) override; const EnumPostingTree& get_posting_dictionary() const override; }; @@ -71,14 +69,14 @@ public: class EnumStoreFoldedDictionary : public EnumStoreDictionary<EnumPostingTree> { private: - std::unique_ptr<vespalib::datastore::EntryComparator> _folded_compare; + std::unique_ptr<EntryComparator> _folded_compare; public: - EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr<vespalib::datastore::EntryComparator> compare, std::unique_ptr<vespalib::datastore::EntryComparator> folded_compare); + EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr<EntryComparator> compare, std::unique_ptr<EntryComparator> folded_compare); ~EnumStoreFoldedDictionary() override; - vespalib::datastore::UniqueStoreAddResult add(const vespalib::datastore::EntryComparator& comp, std::function<vespalib::datastore::EntryRef(void)> insertEntry) override; - void remove(const vespalib::datastore::EntryComparator& comp, vespalib::datastore::EntryRef ref) override; - void collect_folded(Index idx, EntryRef root, const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; + vespalib::datastore::UniqueStoreAddResult add(const EntryComparator& comp, std::function<EntryRef(void)> insertEntry) override; + void remove(const EntryComparator& comp, EntryRef ref) override; + void collect_folded(Index idx, EntryRef root, const std::function<void(EntryRef)>& callback) const override; Index remap_index(Index idx) override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 326e0916039..59d77ea0558 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -63,7 +63,7 @@ private: EnumStoreT(const EnumStoreT & rhs) = delete; EnumStoreT & operator=(const EnumStoreT & rhs) = delete; - void free_value_if_unused(Index idx, IndexSet &unused) override; + void free_value_if_unused(Index idx, IndexList &unused) override; const vespalib::datastore::UniqueStoreEntryBase& get_entry_base(Index idx) const { return _store.get_allocator().get_wrapped(idx); @@ -153,7 +153,7 @@ public: class BatchUpdater { private: EnumStoreType& _store; - IndexSet _possibly_unused; + IndexList _possibly_unused; public: BatchUpdater(EnumStoreType& store) @@ -168,11 +168,11 @@ public: auto& entry = _store.get_entry_base(idx); entry.dec_ref_count(); if (entry.get_ref_count() == 0) { - _possibly_unused.insert(idx); + _possibly_unused.push_back(idx); } } void commit() { - _store.free_unused_values(_possibly_unused); + _store.free_unused_values(std::move(_possibly_unused)); } }; @@ -198,7 +198,7 @@ public: Index insert(EntryType value); bool find_index(EntryType value, Index& idx) const; void free_unused_values() override; - void free_unused_values(const IndexSet& to_remove); + void free_unused_values(IndexList to_remove); vespalib::MemoryUsage update_stat() override; std::unique_ptr<EnumIndexRemapper> consider_compact_values(const CompactionStrategy& compaction_strategy) override; std::unique_ptr<EnumIndexRemapper> compact_worst_values(bool compact_memory, bool compact_address_space) override; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 9885613f4e3..771da8ffa01 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -30,11 +30,11 @@ make_enum_store_dictionary(IEnumStore &store, bool has_postings, const search::D std::unique_ptr<EntryComparator> folded_compare); template <typename EntryT> -void EnumStoreT<EntryT>::free_value_if_unused(Index idx, IndexSet& unused) +void EnumStoreT<EntryT>::free_value_if_unused(Index idx, IndexList& unused) { const auto& entry = get_entry_base(idx); if (entry.get_ref_count() == 0) { - unused.insert(idx); + unused.push_back(idx); _store.get_allocator().hold(idx); } } @@ -140,7 +140,7 @@ EnumStoreT<EntryT>::BatchUpdater::insert(EntryType value) auto cmp = _store.make_comparator(value); auto result = _store._dict->add(cmp, [this, &value]() -> EntryRef { return _store._store.get_allocator().allocate(value); }); if (result.inserted()) { - _possibly_unused.insert(result.ref()); + _possibly_unused.push_back(result.ref()); } return result.ref(); } @@ -191,8 +191,16 @@ EnumStoreT<EntryT>::free_unused_values() template <typename EntryT> void -EnumStoreT<EntryT>::free_unused_values(const IndexSet& to_remove) +EnumStoreT<EntryT>::free_unused_values(IndexList to_remove) { + struct CompareEnumIndex { + using Index = IEnumStore::Index; + + bool operator()(const Index &lhs, const Index &rhs) const { + return lhs.ref() < rhs.ref(); + } + }; + std::sort(to_remove.begin(), to_remove.end(), CompareEnumIndex()); _dict->free_unused_values(to_remove, get_comparator()); } diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index 6d714ec25ba..716609764f4 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -40,22 +40,14 @@ public: using EnumIndexRemapper = vespalib::datastore::UniqueStoreRemapper<InternalIndex>; using Enumerator = vespalib::datastore::UniqueStoreEnumerator<IEnumStore::InternalIndex>; - struct CompareEnumIndex { - using Index = IEnumStore::Index; - - bool operator()(const Index &lhs, const Index &rhs) const { - return lhs.ref() < rhs.ref(); - } - }; - - using IndexSet = std::set<Index, CompareEnumIndex>; + using IndexList = std::vector<Index>; virtual ~IEnumStore() = default; virtual void write_value(BufferWriter& writer, Index idx) const = 0; virtual ssize_t load_unique_values(const void* src, size_t available, IndexVector& idx) = 0; virtual void set_ref_count(Index idx, uint32_t ref_count) = 0; - virtual void free_value_if_unused(Index idx, IndexSet& unused) = 0; + virtual void free_value_if_unused(Index idx, IndexList& unused) = 0; virtual void free_unused_values() = 0; virtual bool is_folded_change(Index idx1, Index idx2) const = 0; virtual IEnumStoreDictionary& get_dictionary() = 0; diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h index f816177b06c..bef7384b0b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h @@ -29,29 +29,29 @@ using EnumPostingTree = vespalib::btree::BTree<IEnumStore::Index, uint32_t, class IEnumStoreDictionary : public vespalib::datastore::IUniqueStoreDictionary { public: using EntryRef = vespalib::datastore::EntryRef; + using EntryComparator = vespalib::datastore::EntryComparator; using EnumVector = IEnumStore::EnumVector; using Index = IEnumStore::Index; - using IndexSet = IEnumStore::IndexSet; + using IndexList = IEnumStore::IndexList; using IndexVector = IEnumStore::IndexVector; using generation_t = vespalib::GenerationHandler::generation_t; public: virtual ~IEnumStoreDictionary() = default; - virtual void free_unused_values(const vespalib::datastore::EntryComparator& cmp) = 0; - virtual void free_unused_values(const IndexSet& to_remove, - const vespalib::datastore::EntryComparator& cmp) = 0; - virtual bool find_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const = 0; - virtual bool find_frozen_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const = 0; + virtual void free_unused_values(const EntryComparator& cmp) = 0; + virtual void free_unused_values(const IndexList& to_remove, const EntryComparator& cmp) = 0; + virtual bool find_index(const EntryComparator& cmp, Index& idx) const = 0; + virtual bool find_frozen_index(const EntryComparator& cmp, Index& idx) const = 0; virtual std::vector<attribute::IAttributeVector::EnumHandle> - find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const = 0; + find_matching_enums(const EntryComparator& cmp) const = 0; virtual EntryRef get_frozen_root() const = 0; - virtual std::pair<Index, EntryRef> find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const = 0; - virtual void collect_folded(Index idx, EntryRef root, const std::function<void(vespalib::datastore::EntryRef)>& callback) const = 0; + virtual std::pair<Index, EntryRef> find_posting_list(const EntryComparator& cmp, EntryRef root) const = 0; + virtual void collect_folded(Index idx, EntryRef root, const std::function<void(EntryRef)>& callback) const = 0; virtual Index remap_index(Index idx) = 0; virtual void clear_all_posting_lists(std::function<void(EntryRef)> clearer) = 0; - virtual void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) = 0; + virtual void update_posting_list(Index idx, const EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) = 0; virtual bool normalize_posting_lists(std::function<EntryRef(EntryRef)> normalize) = 0; virtual const EnumPostingTree& get_posting_dictionary() const = 0; }; diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index bf75400b157..a9a94afb763 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -312,9 +312,9 @@ SingleValueEnumAttribute<B>::onShrinkLidSpace() uint32_t default_value_ref_count = this->_enumStore.get_ref_count(default_value_ref); assert(default_value_ref_count >= shrink_docs); this->_enumStore.set_ref_count(default_value_ref, default_value_ref_count - shrink_docs); - IEnumStore::IndexSet possibly_unused; - possibly_unused.insert(default_value_ref); - this->_enumStore.free_unused_values(possibly_unused); + IEnumStore::IndexList possibly_unused; + possibly_unused.push_back(default_value_ref); + this->_enumStore.free_unused_values(std::move(possibly_unused)); } _enumIndices.shrink(committedDocIdLimit); this->setNumDocs(committedDocIdLimit); diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index 11b4f5e6631..90281acb0d3 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -141,6 +141,38 @@ assertMemStats(const DataStoreBase::MemStats &exp, EXPECT_EQ(exp._holdBuffers, act._holdBuffers); } +TEST(DataStoreTest, require_that_invalid_entry_ref_can_be_ordered) { + EntryRef inValid; + EntryRef a(1); + EXPECT_EQ(inValid, inValid); + EXPECT_EQ(a, a); + EXPECT_NE(inValid, a); + EXPECT_NE(a, inValid); + EXPECT_LT(inValid, a); + EXPECT_LE(inValid, a); +} + +TEST(DataStoreTest, require_that_entry_ref_can_be_ordered) { + EntryRef a(1); + EntryRef b(2); + EntryRef c(3); + EXPECT_EQ(a, a); + EXPECT_EQ(b, b); + EXPECT_EQ(c, c); + EXPECT_NE(a, b); + EXPECT_NE(a, c); + EXPECT_NE(b, c); + EXPECT_LT(a, b); + EXPECT_LT(b, c); + EXPECT_LT(a, c); + EXPECT_LE(a, a); + EXPECT_LE(b, b); + EXPECT_LE(c, c); + EXPECT_LE(a, b); + EXPECT_LE(b, c); + EXPECT_LE(a, c); +} + TEST(DataStoreTest, require_that_entry_ref_is_working) { using MyRefType = EntryRefT<22>; @@ -643,6 +675,7 @@ TEST(DataStoreTest, control_static_sizes) { EXPECT_EQ(0, bs.size()); } + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/datastore/entryref.h b/vespalib/src/vespa/vespalib/datastore/entryref.h index 046d9089580..01f473fcf17 100644 --- a/vespalib/src/vespa/vespalib/datastore/entryref.h +++ b/vespalib/src/vespa/vespalib/datastore/entryref.h @@ -21,6 +21,7 @@ public: bool operator==(const EntryRef &rhs) const noexcept { return _ref == rhs._ref; } bool operator!=(const EntryRef &rhs) const noexcept { return _ref != rhs._ref; } bool operator <(const EntryRef &rhs) const noexcept { return _ref < rhs._ref; } + bool operator <=(const EntryRef &rhs) const noexcept { return _ref <= rhs._ref; } }; /** |