From 573e34dc1ff2f9be5b76c8505fcda6c1b98a2da1 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Thu, 22 Aug 2019 12:49:21 +0000 Subject: Change unique store dictionary to take btree dictionary type as template argument. --- .../src/vespa/vespalib/datastore/CMakeLists.txt | 1 - .../src/vespa/vespalib/datastore/unique_store.hpp | 16 ++- .../vespalib/datastore/unique_store_dictionary.cpp | 134 ------------------ .../vespalib/datastore/unique_store_dictionary.h | 11 +- .../vespalib/datastore/unique_store_dictionary.hpp | 152 +++++++++++++++++++++ 5 files changed, 170 insertions(+), 144 deletions(-) delete mode 100644 vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.cpp create mode 100644 vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp (limited to 'vespalib/src') diff --git a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt index 6f647a78a39..b8c9dae7174 100644 --- a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt @@ -7,7 +7,6 @@ vespa_add_library(vespalib_vespalib_datastore OBJECT datastore.cpp datastorebase.cpp entryref.cpp - unique_store_dictionary.cpp unique_store_string_allocator.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index e149f470bdf..ae7f82a6e52 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -5,20 +5,32 @@ #include "unique_store.h" #include "unique_store_dictionary.h" #include "datastore.hpp" -#include #include "unique_store_allocator.hpp" #include "unique_store_builder.hpp" +#include "unique_store_dictionary.hpp" #include "unique_store_saver.hpp" +#include #include #include namespace search::datastore { +namespace uniquestore { + +using DefaultDictionaryTraits = btree::BTreeTraits<32, 32, 7, true>; +using DefaultDictionary = btree::BTree; +using DefaultUniqueStoreDictionary = UniqueStoreDictionary; + +} + template UniqueStore::UniqueStore() : _allocator(), _store(_allocator.get_data_store()), - _dict(std::make_unique()) + _dict(std::make_unique()) { } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.cpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.cpp deleted file mode 100644 index be28f8f7358..00000000000 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.cpp +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "unique_store_dictionary.h" -#include "unique_store_add_result.h" -#include "entry_comparator_wrapper.h" -#include "i_compactable.h" -#include "datastore.hpp" -#include -#include -#include -#include -#include -#include - -namespace search::datastore { - -UniqueStoreDictionary::UniqueStoreDictionary() - : UniqueStoreDictionaryBase(), - _dict() -{ -} - -UniqueStoreDictionary::~UniqueStoreDictionary() = default; - -void -UniqueStoreDictionary::freeze() -{ - _dict.getAllocator().freeze(); -} - -void -UniqueStoreDictionary::transfer_hold_lists(generation_t generation) -{ - _dict.getAllocator().transferHoldLists(generation); -} - -void -UniqueStoreDictionary::trim_hold_lists(generation_t firstUsed) -{ - _dict.getAllocator().trimHoldLists(firstUsed); -} - -UniqueStoreAddResult -UniqueStoreDictionary::add(const EntryComparator &comp, - std::function insertEntry) -{ - auto itr = _dict.lowerBound(EntryRef(), comp); - if (itr.valid() && !comp(EntryRef(), itr.getKey())) { - return UniqueStoreAddResult(itr.getKey(), false); - - } else { - EntryRef newRef = insertEntry(); - _dict.insert(itr, newRef, btree::BTreeNoLeafData()); - return UniqueStoreAddResult(newRef, true); - } -} - -EntryRef -UniqueStoreDictionary::find(const EntryComparator &comp) -{ - auto itr = _dict.lowerBound(EntryRef(), comp); - if (itr.valid() && !comp(EntryRef(), itr.getKey())) { - return itr.getKey(); - } else { - return EntryRef(); - } -} - -void -UniqueStoreDictionary::remove(const EntryComparator &comp, EntryRef ref) -{ - assert(ref.valid()); - auto itr = _dict.lowerBound(ref, comp); - assert(itr.valid() && itr.getKey() == ref); - _dict.remove(itr); -} - -void -UniqueStoreDictionary::move_entries(ICompactable &compactable) -{ - auto itr = _dict.begin(); - while (itr.valid()) { - EntryRef oldRef(itr.getKey()); - EntryRef newRef(compactable.move(oldRef)); - if (newRef != oldRef) { - _dict.thaw(itr); - itr.writeKey(newRef); - } - ++itr; - } -} - -uint32_t -UniqueStoreDictionary::get_num_uniques() const -{ - return _dict.getFrozenView().size(); -} - -vespalib::MemoryUsage -UniqueStoreDictionary::get_memory_usage() const -{ - return _dict.getMemoryUsage(); -} - -void -UniqueStoreDictionary::build(const std::vector &refs, const std::vector &ref_counts, std::function hold) -{ - assert(refs.size() == ref_counts.size()); - assert(!refs.empty()); - typename Dictionary::Builder builder(_dict.getAllocator()); - for (size_t i = 1; i < refs.size(); ++i) { - if (ref_counts[i] != 0u) { - builder.insert(refs[i], btree::BTreeNoLeafData()); - } else { - hold(refs[i]); - } - } - _dict.assign(builder); -} - -EntryRef -UniqueStoreDictionary::get_frozen_root() const -{ - return _dict.getFrozenView().getRoot(); -} - -void -UniqueStoreDictionary::foreach_key(EntryRef root, std::function callback) const -{ - - _dict.getAllocator().getNodeStore().foreach_key(root, callback); -} - -} diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h index a870d759ce3..c6c73b09259 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h @@ -9,20 +9,17 @@ namespace search::datastore { class EntryComparatorWrapper; -/* +/** * A dictionary for unique store. Mostly accessed via base class. */ +template class UniqueStoreDictionary : public UniqueStoreDictionaryBase { public: - using DictionaryTraits = btree::BTreeTraits<32, 32, 7, true>; - using Dictionary = btree::BTree; + using Dictionary = DictionaryType; private: - Dictionary _dict; + DictionaryType _dict; public: UniqueStoreDictionary(); diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp new file mode 100644 index 00000000000..b73c245956e --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -0,0 +1,152 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "datastore.hpp" +#include "entry_comparator_wrapper.h" +#include "i_compactable.h" +#include "unique_store_add_result.h" +#include "unique_store_dictionary.h" +#include +#include +#include +#include +#include +#include + +namespace search::datastore { + +template +UniqueStoreDictionary::UniqueStoreDictionary() + : UniqueStoreDictionaryBase(), + _dict() +{ +} + +template +UniqueStoreDictionary::~UniqueStoreDictionary() = default; + +template +void +UniqueStoreDictionary::freeze() +{ + _dict.getAllocator().freeze(); +} + +template +void +UniqueStoreDictionary::transfer_hold_lists(generation_t generation) +{ + _dict.getAllocator().transferHoldLists(generation); +} + +template +void +UniqueStoreDictionary::trim_hold_lists(generation_t firstUsed) +{ + _dict.getAllocator().trimHoldLists(firstUsed); +} + +template +UniqueStoreAddResult +UniqueStoreDictionary::add(const EntryComparator &comp, + std::function insertEntry) +{ + auto itr = _dict.lowerBound(EntryRef(), comp); + if (itr.valid() && !comp(EntryRef(), itr.getKey())) { + return UniqueStoreAddResult(itr.getKey(), false); + + } else { + EntryRef newRef = insertEntry(); + _dict.insert(itr, newRef, btree::BTreeNoLeafData()); + return UniqueStoreAddResult(newRef, true); + } +} + +template +EntryRef +UniqueStoreDictionary::find(const EntryComparator &comp) +{ + auto itr = _dict.lowerBound(EntryRef(), comp); + if (itr.valid() && !comp(EntryRef(), itr.getKey())) { + return itr.getKey(); + } else { + return EntryRef(); + } +} + +template +void +UniqueStoreDictionary::remove(const EntryComparator &comp, EntryRef ref) +{ + assert(ref.valid()); + auto itr = _dict.lowerBound(ref, comp); + assert(itr.valid() && itr.getKey() == ref); + _dict.remove(itr); +} + +template +void +UniqueStoreDictionary::move_entries(ICompactable &compactable) +{ + auto itr = _dict.begin(); + while (itr.valid()) { + EntryRef oldRef(itr.getKey()); + EntryRef newRef(compactable.move(oldRef)); + if (newRef != oldRef) { + _dict.thaw(itr); + itr.writeKey(newRef); + } + ++itr; + } +} + +template +uint32_t +UniqueStoreDictionary::get_num_uniques() const +{ + return _dict.getFrozenView().size(); +} + +template +vespalib::MemoryUsage +UniqueStoreDictionary::get_memory_usage() const +{ + return _dict.getMemoryUsage(); +} + +template +void +UniqueStoreDictionary::build(const std::vector &refs, + const std::vector &ref_counts, + std::function hold) +{ + assert(refs.size() == ref_counts.size()); + assert(!refs.empty()); + typename Dictionary::Builder builder(_dict.getAllocator()); + for (size_t i = 1; i < refs.size(); ++i) { + if (ref_counts[i] != 0u) { + builder.insert(refs[i], btree::BTreeNoLeafData()); + } else { + hold(refs[i]); + } + } + _dict.assign(builder); +} + +template +EntryRef +UniqueStoreDictionary::get_frozen_root() const +{ + return _dict.getFrozenView().getRoot(); +} + +template +void +UniqueStoreDictionary::foreach_key(EntryRef root, std::function callback) const +{ + + _dict.getAllocator().getNodeStore().foreach_key(root, callback); +} + +} -- cgit v1.2.3 From ec6553764841765c86a4324802b1d509d1934035 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Thu, 22 Aug 2019 14:15:42 +0000 Subject: Rewrite enum store dictionary to inherit unique store dictionary. --- .../tests/attribute/comparator/comparator_test.cpp | 2 +- .../src/vespa/searchlib/attribute/enumcomparator.h | 18 ++- .../vespa/searchlib/attribute/enumstorebase.cpp | 171 ++++++++------------- .../src/vespa/searchlib/attribute/enumstorebase.h | 127 +++++++-------- .../vespalib/datastore/unique_store_dictionary.h | 4 +- .../vespalib/datastore/unique_store_dictionary.hpp | 4 +- 6 files changed, 130 insertions(+), 196 deletions(-) (limited to 'vespalib/src') diff --git a/searchlib/src/tests/attribute/comparator/comparator_test.cpp b/searchlib/src/tests/attribute/comparator/comparator_test.cpp index 6f16cbbe391..6ad2a7ebe3b 100644 --- a/searchlib/src/tests/attribute/comparator/comparator_test.cpp +++ b/searchlib/src/tests/attribute/comparator/comparator_test.cpp @@ -28,7 +28,7 @@ typedef EnumStoreBase::Index EnumIndex; typedef BTreeRoot TreeType; + const datastore::EntryComparatorWrapper> TreeType; typedef TreeType::NodeAllocatorType NodeAllocator; class Test : public vespalib::TestApp { diff --git a/searchlib/src/vespa/searchlib/attribute/enumcomparator.h b/searchlib/src/vespa/searchlib/attribute/enumcomparator.h index 255d0bead9f..a9e76082a34 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumcomparator.h +++ b/searchlib/src/vespa/searchlib/attribute/enumcomparator.h @@ -10,14 +10,17 @@ namespace search { * Template comparator class for the various entry types. **/ template -class EnumStoreComparatorT : public EnumStoreComparator { +class EnumStoreComparatorT : public datastore::EntryComparator { public: - typedef EnumStoreT EnumStoreType; + using EnumStoreType = EnumStoreT; protected: - typedef typename EntryType::Type EntryValue; + using EntryValue = typename EntryType::Type; + using EnumIndex = typename EnumStoreType::Index; + const EnumStoreType & _enumStore; EntryValue _value; - EntryValue getValue(const EnumIndex & idx) const { + EntryValue getValue(const datastore::EntryRef ref) const { + EnumIndex idx(ref); if (idx.valid()) { return _enumStore.getValue(idx); } @@ -48,7 +51,7 @@ public: } return 1; } - bool operator() (const EnumIndex & lhs, const EnumIndex & rhs) const override { + bool operator() (const datastore::EntryRef lhs, const datastore::EntryRef rhs) const override { return compare(getValue(lhs), getValue(rhs)) < 0; } }; @@ -87,10 +90,11 @@ public: return ParentType::compare(lhs, rhs); } - bool operator() (const EnumIndex & lhs, const EnumIndex & rhs) const override { - if (getUsePrefix()) + bool operator() (const datastore::EntryRef lhs, const datastore::EntryRef rhs) const override { + if (getUsePrefix()) { return compareFoldedPrefix(getValue(lhs), getValue(rhs), _prefixLen) < 0; + } return compareFolded(getValue(lhs), getValue(rhs)) < 0; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp index 5d2523c9875..5825bdd7e58 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp @@ -8,11 +8,12 @@ #include #include #include +#include +#include +#include #include #include #include -#include -#include #include @@ -302,61 +303,42 @@ EnumStoreDictBase::EnumStoreDictBase(EnumStoreBase &enumStore) { } - EnumStoreDictBase::~EnumStoreDictBase() = default; - template EnumStoreDict::EnumStoreDict(EnumStoreBase &enumStore) : EnumStoreDictBase(enumStore), - _dict() + ParentUniqueStoreDictionary() { } template EnumStoreDict::~EnumStoreDict() = default; - -template -void -EnumStoreDict::freezeTree() -{ - _dict.getAllocator().freeze(); -} - template uint32_t EnumStoreDict::getNumUniques() const { - return _dict.size(); -} - - -template -vespalib::MemoryUsage -EnumStoreDict::getTreeMemoryUsage() const -{ - return _dict.getMemoryUsage(); + return this->_dict.size(); } template void EnumStoreDict::reEnumerate() { - _enumStore.reEnumerate(_dict); + _enumStore.reEnumerate(this->_dict); } - template void EnumStoreDict:: writeAllValues(BufferWriter &writer, - btree::BTreeNode::Ref rootRef) const + btree::BTreeNode::Ref rootRef) const { constexpr size_t BATCHSIZE = 1000; std::vector idxs; idxs.reserve(BATCHSIZE); - typename Dictionary::Iterator it(rootRef, _dict.getAllocator()); + typename Dictionary::Iterator it(rootRef, this->_dict.getAllocator()); while (it.valid()) { if (idxs.size() >= idxs.capacity()) { _enumStore.writeValues(writer, &idxs[0], idxs.size()); @@ -370,69 +352,67 @@ writeAllValues(BufferWriter &writer, } } - template ssize_t EnumStoreDict::deserialize(const void *src, - size_t available, - IndexVector &idx) + size_t available, + IndexVector &idx) { - return _enumStore.deserialize(src, available, idx, _dict); + return _enumStore.deserialize(src, available, idx, this->_dict); } - template void EnumStoreDict::fixupRefCounts(const EnumVector & hist) { - _enumStore.fixupRefCounts(hist, _dict); + _enumStore.fixupRefCounts(hist, this->_dict); } - template void EnumStoreDict::removeUnusedEnums(const IndexSet &unused, - const EnumStoreComparator &cmp, - const EnumStoreComparator *fcmp) + const datastore::EntryComparator &cmp, + const datastore::EntryComparator *fcmp) { typedef typename Dictionary::Iterator Iterator; if (unused.empty()) return; - Iterator it(BTreeNode::Ref(), _dict.getAllocator()); - for (IndexSet::const_iterator iter(unused.begin()), mt(unused.end()); - iter != mt; ++iter) { - it.lower_bound(_dict.getRoot(), *iter, cmp); - assert(it.valid() && !cmp(*iter, it.getKey())); + Iterator it(BTreeNode::Ref(), this->_dict.getAllocator()); + for (const auto& idx : unused) { + it.lower_bound(this->_dict.getRoot(), idx, cmp); + assert(it.valid() && !cmp(idx, it.getKey())); if (Iterator::hasData() && fcmp != nullptr) { typename Dictionary::DataType pidx(it.getData()); - _dict.remove(it); - if (!it.valid() || (*fcmp)(*iter, it.getKey())) + this->_dict.remove(it); + if (!it.valid() || (*fcmp)(idx, it.getKey())) { continue; // Next entry does not use same posting list + } --it; - if (it.valid() && !(*fcmp)(it.getKey(), *iter)) + if (it.valid() && !(*fcmp)(it.getKey(), idx)) { continue; // Previous entry uses same posting list - if (it.valid()) + } + if (it.valid()) { ++it; - else + } else { it.begin(); - _dict.thaw(it); + } + this->_dict.thaw(it); it.writeData(pidx); } else { - _dict.remove(it); + this->_dict.remove(it); } } } template void -EnumStoreDict::freeUnusedEnums(const EnumStoreComparator &cmp, - const EnumStoreComparator *fcmp) +EnumStoreDict::freeUnusedEnums(const datastore::EntryComparator &cmp, + const datastore::EntryComparator *fcmp) { IndexSet unused; // find unused enums - for (typename Dictionary::Iterator iter(_dict.begin()); iter.valid(); - ++iter) { + for (auto iter = this->_dict.begin(); iter.valid(); ++iter) { _enumStore.freeUnusedEnum(iter.getKey(), unused); } removeUnusedEnums(unused, cmp, fcmp); @@ -441,8 +421,8 @@ EnumStoreDict::freeUnusedEnums(const EnumStoreComparator &cmp, template void EnumStoreDict::freeUnusedEnums(const IndexSet& toRemove, - const EnumStoreComparator& cmp, - const EnumStoreComparator* fcmp) + const datastore::EntryComparator& cmp, + const datastore::EntryComparator* fcmp) { IndexSet unused; for (const auto& index : toRemove) { @@ -451,13 +431,12 @@ EnumStoreDict::freeUnusedEnums(const IndexSet& toRemove, removeUnusedEnums(unused, cmp, fcmp); } - template bool -EnumStoreDict::findIndex(const EnumStoreComparator &cmp, +EnumStoreDict::findIndex(const datastore::EntryComparator &cmp, Index &idx) const { - typename Dictionary::Iterator itr = _dict.find(Index(), cmp); + auto itr = this->_dict.find(Index(), cmp); if (!itr.valid()) { return false; } @@ -465,14 +444,12 @@ EnumStoreDict::findIndex(const EnumStoreComparator &cmp, return true; } - template bool -EnumStoreDict::findFrozenIndex(const EnumStoreComparator &cmp, +EnumStoreDict::findFrozenIndex(const datastore::EntryComparator &cmp, Index &idx) const { - typename Dictionary::ConstIterator itr = - _dict.getFrozenView().find(Index(), cmp); + auto itr = this->_dict.getFrozenView().find(Index(), cmp); if (!itr.valid()) { return false; } @@ -480,14 +457,12 @@ EnumStoreDict::findFrozenIndex(const EnumStoreComparator &cmp, return true; } - template std::vector -EnumStoreDict::findMatchingEnums(const EnumStoreComparator &cmp) const +EnumStoreDict::findMatchingEnums(const datastore::EntryComparator &cmp) const { std::vector result; - typename Dictionary::ConstIterator itr = - _dict.getFrozenView().find(Index(), cmp); + auto itr = this->_dict.getFrozenView().find(Index(), cmp); while (itr.valid() && !cmp(Index(), itr.getKey())) { result.push_back(itr.getKey().ref()); ++itr; @@ -499,42 +474,16 @@ template void EnumStoreDict::onReset() { - _dict.clear(); -} - - -template -void -EnumStoreDict::onTransferHoldLists(generation_t generation) -{ - _dict.getAllocator().transferHoldLists(generation); + this->_dict.clear(); } - -template -void -EnumStoreDict::onTrimHoldLists(generation_t firstUsed) -{ - _dict.getAllocator().trimHoldLists(firstUsed); -} - - -template -BTreeNode::Ref -EnumStoreDict::getFrozenRootRef() const -{ - return _dict.getFrozenView().getRoot(); -} - - template uint32_t -EnumStoreDict:: -lookupFrozenTerm(BTreeNode::Ref frozenRootRef, - const EnumStoreComparator &comp) const +EnumStoreDict::lookupFrozenTerm(BTreeNode::Ref frozenRootRef, + const datastore::EntryComparator &comp) const { typename Dictionary::ConstIterator itr(BTreeNode::Ref(), - _dict.getAllocator()); + this->_dict.getAllocator()); itr.lower_bound(frozenRootRef, Index(), comp); if (itr.valid() && !comp(Index(), itr.getKey())) { return 1u; @@ -542,20 +491,20 @@ lookupFrozenTerm(BTreeNode::Ref frozenRootRef, return 0u; } - template uint32_t EnumStoreDict:: lookupFrozenRange(BTreeNode::Ref frozenRootRef, - const EnumStoreComparator &low, - const EnumStoreComparator &high) const + const datastore::EntryComparator &low, + const datastore::EntryComparator &high) const { typename Dictionary::ConstIterator lowerDictItr(BTreeNode::Ref(), - _dict.getAllocator()); + this->_dict.getAllocator()); lowerDictItr.lower_bound(frozenRootRef, Index(), low); - typename Dictionary::ConstIterator upperDictItr = lowerDictItr; - if (upperDictItr.valid() && !high(Index(), upperDictItr.getKey())) + auto upperDictItr = lowerDictItr; + if (upperDictItr.valid() && !high(Index(), upperDictItr.getKey())) { upperDictItr.seekPast(Index(), high); + } return upperDictItr - lowerDictItr; } @@ -665,19 +614,19 @@ class btree::BTreeNodeStore; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; template class btree::BTreeRoot; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; template class btree::BTreeRootT; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; template class btree::BTreeRootT; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; template class btree::BTreeRootBase; -template class btree::BTreeConstIterator; +template class btree::BTreeConstIterator; -template class btree::BTreeConstIterator; +template class btree::BTreeConstIterator; template class btree::BTreeIterator; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; template class btree::BTreeIterator; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; template class btree::BTree; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; template class btree::BTree; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; } diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h index 3316db534fa..ea6ace1438a 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h @@ -5,10 +5,13 @@ #include #include #include +#include +#include +#include +#include #include #include #include -#include #include #include @@ -20,9 +23,8 @@ class BufferWriter; namespace attribute { class Status; } class EnumStoreBase; -class EnumStoreComparator; -class EnumStoreComparatorWrapper; +using EnumStoreComparator = datastore::EntryComparator; using EnumStoreDataStoreType = datastore::DataStoreT >; using EnumStoreIndex = EnumStoreDataStoreType::RefType; using EnumStoreIndexVector = vespalib::Array; @@ -32,11 +34,11 @@ typedef btree::BTreeTraits<16, 16, 10, true> EnumTreeTraits; typedef btree::BTree EnumTree; typedef btree::BTree EnumPostingTree; struct CompareEnumIndex @@ -72,15 +74,15 @@ public: virtual ssize_t deserialize(const void *src, size_t available, IndexVector &idx) = 0; virtual void fixupRefCounts(const EnumVector &hist) = 0; - virtual void freeUnusedEnums(const EnumStoreComparator &cmp, - const EnumStoreComparator *fcmp) = 0; + virtual void freeUnusedEnums(const datastore::EntryComparator &cmp, + const datastore::EntryComparator *fcmp) = 0; virtual void freeUnusedEnums(const IndexSet& toRemove, - const EnumStoreComparator& cmp, - const EnumStoreComparator* fcmp) = 0; - virtual bool findIndex(const EnumStoreComparator &cmp, Index &idx) const = 0; - virtual bool findFrozenIndex(const EnumStoreComparator &cmp, Index &idx) const = 0; + const datastore::EntryComparator& cmp, + const datastore::EntryComparator* fcmp) = 0; + virtual bool findIndex(const datastore::EntryComparator &cmp, Index &idx) const = 0; + virtual bool findFrozenIndex(const datastore::EntryComparator &cmp, Index &idx) const = 0; virtual std::vector - findMatchingEnums(const EnumStoreComparator &cmp) const = 0; + findMatchingEnums(const datastore::EntryComparator &cmp) const = 0; virtual void onReset() = 0; virtual void onTransferHoldLists(generation_t generation) = 0; @@ -88,11 +90,11 @@ public: virtual btree::BTreeNode::Ref getFrozenRootRef() const = 0; virtual uint32_t lookupFrozenTerm(btree::BTreeNode::Ref frozenRootRef, - const EnumStoreComparator &comp) const = 0; + const datastore::EntryComparator &comp) const = 0; virtual uint32_t lookupFrozenRange(btree::BTreeNode::Ref frozenRootRef, - const EnumStoreComparator &low, - const EnumStoreComparator &high) const = 0; + const datastore::EntryComparator &low, + const datastore::EntryComparator &high) const = 0; virtual EnumPostingTree &getPostingDictionary() = 0; virtual const EnumPostingTree &getPostingDictionary() const = 0; @@ -101,54 +103,55 @@ public: template -class EnumStoreDict : public EnumStoreDictBase +class EnumStoreDict : public EnumStoreDictBase, + search::datastore::UniqueStoreDictionary { -protected: - Dictionary _dict; +private: + using ParentUniqueStoreDictionary = search::datastore::UniqueStoreDictionary; public: EnumStoreDict(EnumStoreBase &enumStore); ~EnumStoreDict() override; - const Dictionary &getDictionary() const { return _dict; } - Dictionary &getDictionary() { return _dict; } + const Dictionary &getDictionary() const { return this->_dict; } + Dictionary &getDictionary() { return this->_dict; } - void freezeTree() override; + void freezeTree() override { this->freeze(); } uint32_t getNumUniques() const override; - vespalib::MemoryUsage getTreeMemoryUsage() const override; + vespalib::MemoryUsage getTreeMemoryUsage() const override { return this->get_memory_usage(); } void reEnumerate() override; void writeAllValues(BufferWriter &writer, btree::BTreeNode::Ref rootRef) const override; ssize_t deserialize(const void *src, size_t available, IndexVector &idx) override; void fixupRefCounts(const EnumVector &hist) override; void removeUnusedEnums(const IndexSet &unused, - const EnumStoreComparator &cmp, - const EnumStoreComparator *fcmp); + const datastore::EntryComparator &cmp, + const datastore::EntryComparator *fcmp); - void freeUnusedEnums(const EnumStoreComparator &cmp, - const EnumStoreComparator *fcmp) override; + void freeUnusedEnums(const datastore::EntryComparator &cmp, + const datastore::EntryComparator *fcmp) override; void freeUnusedEnums(const IndexSet& toRemove, - const EnumStoreComparator& cmp, - const EnumStoreComparator* fcmp) override; + const datastore::EntryComparator& cmp, + const datastore::EntryComparator* fcmp) override; - bool findIndex(const EnumStoreComparator &cmp, Index &idx) const override; - bool findFrozenIndex(const EnumStoreComparator &cmp, Index &idx) const override; + bool findIndex(const datastore::EntryComparator &cmp, Index &idx) const override; + bool findFrozenIndex(const datastore::EntryComparator &cmp, Index &idx) const override; std::vector - findMatchingEnums(const EnumStoreComparator &cmp) const override; + findMatchingEnums(const datastore::EntryComparator &cmp) const override; void onReset() override; - void onTransferHoldLists(generation_t generation) override; - void onTrimHoldLists(generation_t firstUsed) override; - btree::BTreeNode::Ref getFrozenRootRef() const override; + void onTransferHoldLists(generation_t generation) override { this->transfer_hold_lists(generation); } + void onTrimHoldLists(generation_t firstUsed) override { this->trim_hold_lists(firstUsed); } + btree::BTreeNode::Ref getFrozenRootRef() const override { return this->get_frozen_root(); } uint32_t lookupFrozenTerm(btree::BTreeNode::Ref frozenRootRef, - const EnumStoreComparator &comp) const override; + const datastore::EntryComparator &comp) const override; uint32_t lookupFrozenRange(btree::BTreeNode::Ref frozenRootRef, - const EnumStoreComparator &low, - const EnumStoreComparator &high) const override; + const datastore::EntryComparator &low, + const datastore::EntryComparator &high) const override; EnumPostingTree & getPostingDictionary() override; const EnumPostingTree & getPostingDictionary() const override; @@ -359,34 +362,6 @@ public: vespalib::asciistream & operator << (vespalib::asciistream & os, const EnumStoreBase::Index & idx); -/** - * Base comparator class needed by the btree. - **/ -class EnumStoreComparator { -public: - typedef EnumStoreBase::Index EnumIndex; - virtual ~EnumStoreComparator() {} - /** - * Compare the values represented by the given enum indexes. - * Uses the enum store to map from enum index to actual value. - **/ - virtual bool operator() (const EnumIndex & lhs, const EnumIndex & rhs) const = 0; -}; - - -class EnumStoreComparatorWrapper -{ - const EnumStoreComparator &_comp; -public: - typedef EnumStoreBase::Index EnumIndex; - EnumStoreComparatorWrapper(const EnumStoreComparator &comp) - : _comp(comp) - { } - - bool operator()(const EnumIndex &lhs, const EnumIndex &rhs) const { - return _comp(lhs, rhs); - } -}; extern template class datastore::DataStoreT >; @@ -425,19 +400,19 @@ class btree::BTreeNodeStore; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template class btree::BTreeRoot; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template class btree::BTreeRootT; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template class btree::BTreeRootT; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template class btree::BTreeRootBase; -extern template class btree::BTreeConstIterator; +extern template class btree::BTreeConstIterator; -extern template class btree::BTreeConstIterator; +extern template class btree::BTreeConstIterator; extern template class btree::BTreeIterator; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template class btree::BTreeIterator; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template class btree::BTree; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template class btree::BTree; + const datastore::EntryComparatorWrapper, EnumTreeTraits>; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h index c6c73b09259..c09aab25c69 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h @@ -18,7 +18,9 @@ class UniqueStoreDictionary : public UniqueStoreDictionaryBase public: using Dictionary = DictionaryType; -private: +protected: + using DataType = typename DictionaryType::DataType; + DictionaryType _dict; public: diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp index b73c245956e..d4a275cd12e 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -58,7 +58,7 @@ UniqueStoreDictionary::add(const EntryComparator &comp, } else { EntryRef newRef = insertEntry(); - _dict.insert(itr, newRef, btree::BTreeNoLeafData()); + _dict.insert(itr, newRef, DataType()); return UniqueStoreAddResult(newRef, true); } } @@ -126,7 +126,7 @@ UniqueStoreDictionary::build(const std::vector &refs, typename Dictionary::Builder builder(_dict.getAllocator()); for (size_t i = 1; i < refs.size(); ++i) { if (ref_counts[i] != 0u) { - builder.insert(refs[i], btree::BTreeNoLeafData()); + builder.insert(refs[i], DataType()); } else { hold(refs[i]); } -- cgit v1.2.3 From 32a1d842d2aaf08ea996c9b142ef89aca1b7b3f0 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Fri, 23 Aug 2019 07:01:31 +0000 Subject: Rename template argument. --- .../vespalib/datastore/unique_store_dictionary.h | 6 +- .../vespalib/datastore/unique_store_dictionary.hpp | 65 +++++++++++----------- 2 files changed, 34 insertions(+), 37 deletions(-) (limited to 'vespalib/src') diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h index c09aab25c69..4191b5d26a6 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h @@ -12,13 +12,11 @@ class EntryComparatorWrapper; /** * A dictionary for unique store. Mostly accessed via base class. */ -template +template class UniqueStoreDictionary : public UniqueStoreDictionaryBase { -public: - using Dictionary = DictionaryType; - protected: + using DictionaryType = DictionaryT; using DataType = typename DictionaryType::DataType; DictionaryType _dict; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp index d4a275cd12e..51968f0042b 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -16,41 +16,41 @@ namespace search::datastore { -template -UniqueStoreDictionary::UniqueStoreDictionary() +template +UniqueStoreDictionary::UniqueStoreDictionary() : UniqueStoreDictionaryBase(), _dict() { } -template -UniqueStoreDictionary::~UniqueStoreDictionary() = default; +template +UniqueStoreDictionary::~UniqueStoreDictionary() = default; -template +template void -UniqueStoreDictionary::freeze() +UniqueStoreDictionary::freeze() { _dict.getAllocator().freeze(); } -template +template void -UniqueStoreDictionary::transfer_hold_lists(generation_t generation) +UniqueStoreDictionary::transfer_hold_lists(generation_t generation) { _dict.getAllocator().transferHoldLists(generation); } -template +template void -UniqueStoreDictionary::trim_hold_lists(generation_t firstUsed) +UniqueStoreDictionary::trim_hold_lists(generation_t firstUsed) { _dict.getAllocator().trimHoldLists(firstUsed); } -template +template UniqueStoreAddResult -UniqueStoreDictionary::add(const EntryComparator &comp, - std::function insertEntry) +UniqueStoreDictionary::add(const EntryComparator &comp, + std::function insertEntry) { auto itr = _dict.lowerBound(EntryRef(), comp); if (itr.valid() && !comp(EntryRef(), itr.getKey())) { @@ -63,9 +63,9 @@ UniqueStoreDictionary::add(const EntryComparator &comp, } } -template +template EntryRef -UniqueStoreDictionary::find(const EntryComparator &comp) +UniqueStoreDictionary::find(const EntryComparator &comp) { auto itr = _dict.lowerBound(EntryRef(), comp); if (itr.valid() && !comp(EntryRef(), itr.getKey())) { @@ -75,9 +75,9 @@ UniqueStoreDictionary::find(const EntryComparator &comp) } } -template +template void -UniqueStoreDictionary::remove(const EntryComparator &comp, EntryRef ref) +UniqueStoreDictionary::remove(const EntryComparator &comp, EntryRef ref) { assert(ref.valid()); auto itr = _dict.lowerBound(ref, comp); @@ -85,9 +85,9 @@ UniqueStoreDictionary::remove(const EntryComparator &comp, Entry _dict.remove(itr); } -template +template void -UniqueStoreDictionary::move_entries(ICompactable &compactable) +UniqueStoreDictionary::move_entries(ICompactable &compactable) { auto itr = _dict.begin(); while (itr.valid()) { @@ -101,29 +101,29 @@ UniqueStoreDictionary::move_entries(ICompactable &compactable) } } -template +template uint32_t -UniqueStoreDictionary::get_num_uniques() const +UniqueStoreDictionary::get_num_uniques() const { return _dict.getFrozenView().size(); } -template +template vespalib::MemoryUsage -UniqueStoreDictionary::get_memory_usage() const +UniqueStoreDictionary::get_memory_usage() const { return _dict.getMemoryUsage(); } -template +template void -UniqueStoreDictionary::build(const std::vector &refs, - const std::vector &ref_counts, - std::function hold) +UniqueStoreDictionary::build(const std::vector &refs, + const std::vector &ref_counts, + std::function hold) { assert(refs.size() == ref_counts.size()); assert(!refs.empty()); - typename Dictionary::Builder builder(_dict.getAllocator()); + typename DictionaryType::Builder builder(_dict.getAllocator()); for (size_t i = 1; i < refs.size(); ++i) { if (ref_counts[i] != 0u) { builder.insert(refs[i], DataType()); @@ -134,18 +134,17 @@ UniqueStoreDictionary::build(const std::vector &refs, _dict.assign(builder); } -template +template EntryRef -UniqueStoreDictionary::get_frozen_root() const +UniqueStoreDictionary::get_frozen_root() const { return _dict.getFrozenView().getRoot(); } -template +template void -UniqueStoreDictionary::foreach_key(EntryRef root, std::function callback) const +UniqueStoreDictionary::foreach_key(EntryRef root, std::function callback) const { - _dict.getAllocator().getNodeStore().foreach_key(root, callback); } -- cgit v1.2.3