diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-02-11 21:06:07 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-02-11 21:06:07 +0100 |
commit | 30cdbc3cad0279e6d6c9bc337926598046ab09a6 (patch) | |
tree | b597a11ff6aaf31057527855ddbb666c86d183cc | |
parent | 7dcada4f7521ec2fc5224e693ffa46744246b60c (diff) | |
parent | 2e851fa9928b85a7328751d26af1db564430edfe (diff) |
Merge pull request #25998 from vespa-engine/balder/reduce-inlining
Balder/reduce inlining
9 files changed, 170 insertions, 152 deletions
diff --git a/eval/src/vespa/eval/eval/fast_addr_map.h b/eval/src/vespa/eval/eval/fast_addr_map.h index bda708b0945..442e0560b4b 100644 --- a/eval/src/vespa/eval/eval/fast_addr_map.h +++ b/eval/src/vespa/eval/eval/fast_addr_map.h @@ -20,13 +20,13 @@ class FastAddrMap { public: // label extracting functions - static constexpr string_id self(string_id label) { return label; } - static constexpr string_id self(const string_id *label) { return *label; } + static constexpr string_id self(string_id label) noexcept { return label; } + static constexpr string_id self(const string_id *label) noexcept { return *label; } // label hashing functions - static constexpr uint32_t hash_label(string_id label) { return label.value(); } - static constexpr uint32_t hash_label(const string_id *label) { return label->value(); } - static constexpr uint32_t combine_label_hash(uint32_t full_hash, uint32_t next_hash) { + static constexpr uint32_t hash_label(string_id label) noexcept { return label.value(); } + static constexpr uint32_t hash_label(const string_id *label) noexcept { return label->value(); } + static constexpr uint32_t combine_label_hash(uint32_t full_hash, uint32_t next_hash) noexcept { return ((full_hash * 31) + next_hash); } template <typename T> @@ -41,9 +41,9 @@ public: // typed uint32_t index used to identify sparse address/dense subspace struct Tag { uint32_t idx; - static constexpr uint32_t npos() { return uint32_t(-1); } - static constexpr Tag make_invalid() { return Tag{npos()}; } - constexpr bool valid() const { return (idx != npos()); } + static constexpr uint32_t npos() noexcept { return uint32_t(-1); } + static constexpr Tag make_invalid() noexcept { return Tag{npos()}; } + constexpr bool valid() const noexcept { return (idx != npos()); } }; // sparse hash set entry @@ -62,9 +62,9 @@ public: struct LabelView { size_t addr_size; const StringIdVector &labels; - LabelView(size_t num_mapped_dims, const StringIdVector &labels_in) + LabelView(size_t num_mapped_dims, const StringIdVector &labels_in) noexcept : addr_size(num_mapped_dims), labels(labels_in) {} - ConstArrayRef<string_id> get_addr(size_t idx) const { + ConstArrayRef<string_id> get_addr(size_t idx) const noexcept { return {labels.data() + (idx * addr_size), addr_size}; } }; @@ -72,17 +72,17 @@ public: // hashing functor for sparse hash set struct Hash { template <typename T> - constexpr uint32_t operator()(const AltKey<T> &key) const { return key.hash; } - constexpr uint32_t operator()(const Entry &entry) const { return entry.hash; } - constexpr uint32_t operator()(string_id label) const { return label.value(); } + constexpr uint32_t operator()(const AltKey<T> &key) const noexcept { return key.hash; } + constexpr uint32_t operator()(const Entry &entry) const noexcept { return entry.hash; } + constexpr uint32_t operator()(string_id label) const noexcept { return label.value(); } }; // equality functor for sparse hash set struct Equal { const LabelView &label_view; - Equal(const LabelView &label_view_in) : label_view(label_view_in) {} + Equal(const LabelView &label_view_in) noexcept : label_view(label_view_in) {} template <typename T> - bool operator()(const Entry &a, const AltKey<T> &b) const { + bool operator()(const Entry &a, const AltKey<T> &b) const noexcept { if (a.hash != b.hash) { return false; } @@ -110,25 +110,25 @@ public: FastAddrMap &operator=(const FastAddrMap &) = delete; FastAddrMap(FastAddrMap &&) = delete; FastAddrMap &operator=(FastAddrMap &&) = delete; - static constexpr size_t npos() { return -1; } - ConstArrayRef<string_id> get_addr(size_t idx) const { return _labels.get_addr(idx); } - size_t size() const { return _map.size(); } - constexpr size_t addr_size() const { return _labels.addr_size; } - const StringIdVector &labels() const { return _labels.labels; } + static constexpr size_t npos() noexcept { return -1; } + ConstArrayRef<string_id> get_addr(size_t idx) const noexcept { return _labels.get_addr(idx); } + size_t size() const noexcept { return _map.size(); } + constexpr size_t addr_size() const noexcept { return _labels.addr_size; } + const StringIdVector &labels() const noexcept { return _labels.labels; } template <typename T> - size_t lookup(ConstArrayRef<T> addr, uint32_t hash) const { + size_t lookup(ConstArrayRef<T> addr, uint32_t hash) const noexcept { // assert(addr_size() == addr.size()); AltKey<T> key{addr, hash}; auto pos = _map.find(key); return (pos == _map.end()) ? npos() : pos->tag.idx; } - size_t lookup_singledim(string_id addr) const { + size_t lookup_singledim(string_id addr) const noexcept { // assert(addr_size() == 1); auto pos = _map.find(addr); return (pos == _map.end()) ? npos() : pos->tag.idx; } template <typename T> - size_t lookup(ConstArrayRef<T> addr) const { + size_t lookup(ConstArrayRef<T> addr) const noexcept { return (addr.size() == 1) ? lookup_singledim(self(addr[0])) : lookup(addr, hash_labels(addr)); diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 7f2b553e19d..0c0086861b2 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -31,7 +31,23 @@ using storage::spi::test::equal; namespace storage::spi::dummy { -BucketContent::BucketContent() +namespace { + +std::string asString(const std::vector<BucketEntry> &) __attribute__((noinline)); + +std::string +asString(const std::vector<BucketEntry> & v) { + vespalib::asciistream ost; + for (const auto & e : v) { + ost << e.entry->toString() << "\n"; + } + + return ost.str(); +} + +} + +BucketContent::BucketContent() noexcept : _entries(), _gidMap(), _info(), @@ -300,16 +316,18 @@ DummyPersistence::initialize() { return Result(); } -#define DUMMYPERSISTENCE_VERIFY_INITIALIZED \ - if (!_initialized) { \ - LOG(error, "initialize() must always be called first in order to trigger lazy initialization."); \ - abort(); \ +void +DummyPersistence::verifyInitialized() const noexcept { + if (!_initialized) { + LOG(error, "initialize() must always be called first in order to trigger lazy initialization."); + abort(); } +} BucketIdListResult DummyPersistence::listBuckets(BucketSpace bucketSpace) const { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(debug, "listBuckets()"); std::lock_guard lock(_monitor); BucketIdListResult::List list; @@ -373,7 +391,7 @@ DummyPersistence::setClusterState(BucketSpace bucketSpace, const ClusterState& c void DummyPersistence::setActiveStateAsync(const Bucket& b, BucketInfo::ActiveState newState, OperationComplete::UP onComplete) { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(debug, "setCurrentState(%s, %s)", b.toString().c_str(), newState == BucketInfo::ACTIVE ? "ACTIVE" : "INACTIVE"); @@ -395,7 +413,7 @@ DummyPersistence::setActiveStateAsync(const Bucket& b, BucketInfo::ActiveState n BucketInfoResult DummyPersistence::getBucketInfo(const Bucket& b) const { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { @@ -415,7 +433,7 @@ DummyPersistence::getBucketInfo(const Bucket& b) const void DummyPersistence::putAsync(const Bucket& b, Timestamp t, Document::SP doc, OperationComplete::UP onComplete) { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(debug, "put(%s, %" PRIu64 ", %s)", b.toString().c_str(), uint64_t(t), doc->getId().toString().c_str()); assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); @@ -478,7 +496,7 @@ DummyPersistence::updateAsync(const Bucket& bucket, Timestamp ts, DocumentUpdate void DummyPersistence::removeAsync(const Bucket& b, std::vector<spi::IdAndTimestamp> ids, OperationComplete::UP onComplete) { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); @@ -514,7 +532,7 @@ DummyPersistence::removeAsync(const Bucket& b, std::vector<spi::IdAndTimestamp> GetResult DummyPersistence::get(const Bucket& b, const FieldSet& fieldSet, const DocumentId& did, Context&) const { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(debug, "get(%s, %s)", b.toString().c_str(), did.toString().c_str()); @@ -542,7 +560,7 @@ DummyPersistence::get(const Bucket& b, const FieldSet& fieldSet, const DocumentI CreateIteratorResult DummyPersistence::createIterator(const Bucket &b, FieldSetSP fs, const Selection &s, IncludedVersions v, Context &) { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(debug, "createIterator(%s)", b.toString().c_str()); assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); std::unique_ptr<document::select::Node> docSelection; @@ -628,7 +646,7 @@ DummyPersistence::createIterator(const Bucket &b, FieldSetSP fs, const Selection IterateResult DummyPersistence::iterate(IteratorId id, uint64_t maxByteSize) const { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(debug, "iterate(%" PRIu64 ", %" PRIu64 ")", uint64_t(id), maxByteSize); Iterator* it; { @@ -691,7 +709,7 @@ DummyPersistence::iterate(IteratorId id, uint64_t maxByteSize) const Result DummyPersistence::destroyIterator(IteratorId id) { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(debug, "destroyIterator(%" PRIu64 ")", uint64_t(id)); std::lock_guard lock(_monitor); if (_iterators.find(id) != _iterators.end()) { @@ -703,11 +721,11 @@ DummyPersistence::destroyIterator(IteratorId id) void DummyPersistence::createBucketAsync(const Bucket& b, OperationComplete::UP onComplete) noexcept { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(debug, "createBucket(%s)", b.toString().c_str()); assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); std::lock_guard lock(_monitor); - if (_content.find(b) == _content.end()) { + if (find(b) == _content.end()) { _content[b] = std::make_shared<BucketContent>(); } else { assert(!_content[b]->_inUse); @@ -719,7 +737,7 @@ DummyPersistence::createBucketAsync(const Bucket& b, OperationComplete::UP onCom void DummyPersistence::deleteBucketAsync(const Bucket& b, OperationComplete::UP onComplete) noexcept { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(debug, "deleteBucket(%s)", b.toString().c_str()); assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); std::lock_guard lock(_monitor); @@ -733,7 +751,7 @@ DummyPersistence::deleteBucketAsync(const Bucket& b, OperationComplete::UP onCom Result DummyPersistence::split(const Bucket& source, const Bucket& target1, const Bucket& target2) { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(debug, "split(%s -> %s, %s)", source.toString().c_str(), target1.toString().c_str(), @@ -788,7 +806,7 @@ DummyPersistence::split(const Bucket& source, const Bucket& target1, const Bucke Result DummyPersistence::join(const Bucket& source1, const Bucket& source2, const Bucket& target) { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(debug, "join(%s, %s -> %s)", source1.toString().c_str(), source2.toString().c_str(), @@ -854,32 +872,22 @@ DummyPersistence::register_executor(std::shared_ptr<BucketExecutor> executor) std::string DummyPersistence::dumpBucket(const Bucket& b) const { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); LOG(spam, "dumpBucket(%s)", b.toString().c_str()); assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); std::lock_guard lock(_monitor); - Content::const_iterator it(_content.find(b)); - if (it == _content.end()) { - return "DOESN'T EXIST"; - } else { - vespalib::asciistream ost; - for (uint32_t i=0; i<it->second->_entries.size(); ++i) { - const DocEntry& entry(*it->second->_entries[i].entry); - ost << entry.toString() << "\n"; - } - - return ost.str(); - } + auto it = find(b); + return (it != _content.end()) ? asString(it->second->_entries) : "DOESN'T EXIST"; } bool DummyPersistence::isActive(const Bucket& b) const { - DUMMYPERSISTENCE_VERIFY_INITIALIZED; + verifyInitialized(); assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); std::lock_guard lock(_monitor); LOG(spam, "isActive(%s)", b.toString().c_str()); - Content::const_iterator it(_content.find(b)); + auto it(find(b)); if (it == _content.end()) { return false; } @@ -897,7 +905,7 @@ DummyPersistence::acquireBucketWithLock(const Bucket& b, LockMode lock_mode) con assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); std::lock_guard lock(_monitor); DummyPersistence& ncp(const_cast<DummyPersistence&>(*this)); - Content::iterator it(ncp._content.find(b)); + auto it(ncp.find(b)); if (it == ncp._content.end()) { return BucketContentGuard::UP(); } @@ -933,11 +941,16 @@ void DummyPersistence::internal_create_bucket(const Bucket& b) { std::lock_guard lock(_monitor); - if (_content.find(b) == _content.end()) { + if (find(b) == _content.end()) { _content[b] = std::make_shared<BucketContent>(); } } +DummyPersistence::Content::const_iterator +DummyPersistence::find(const Bucket & bucket) const { + return _content.find(bucket); +} + } VESPALIB_HASH_MAP_INSTANTIATE_H(storage::spi::Bucket, std::shared_ptr<storage::spi::dummy::BucketContent>, document::BucketId::hash) diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index 56602b3ab00..859c8687520 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -36,7 +36,7 @@ struct BucketEntry DocEntry::SP entry; GlobalId gid; - BucketEntry(DocEntry::SP e, const GlobalId& g) + BucketEntry(DocEntry::SP e, const GlobalId& g) noexcept : entry(std::move(e)), gid(g) { } @@ -58,7 +58,7 @@ struct BucketContent { mutable bool _outdatedInfo; bool _active; - BucketContent(); + BucketContent() noexcept; ~BucketContent(); @@ -200,6 +200,7 @@ public: } private: + void verifyInitialized() const noexcept __attribute__((noinline)); friend class BucketContentGuard; // Const since funcs only alter mutable field in BucketContent BucketContentGuard::UP acquireBucketWithLock(const Bucket& b, LockMode lock_mode = LockMode::Exclusive) const; @@ -219,11 +220,9 @@ private: std::unique_ptr<ClusterState> _clusterState; std::weak_ptr<BucketExecutor> _bucket_executor; - std::unique_ptr<document::select::Node> parseDocumentSelection( - const string& documentSelection, - bool allowLeaf); - mutable BucketIdListResult::List _modifiedBuckets; + std::unique_ptr<document::select::Node> parseDocumentSelection(const string& documentSelection, bool allowLeaf); + Content::const_iterator find(const Bucket & bucket) const __attribute__((noinline)); }; } diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.h b/vespalib/src/vespa/vespalib/stllike/hash_map.h index 48448404d9b..889093a9550 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.h @@ -31,13 +31,13 @@ public: hash_map(size_t reserveSize, H hasher, EQ equality); hash_map(std::initializer_list<value_type> input); ~hash_map() noexcept; - iterator begin() { return _ht.begin(); } - iterator end() { return _ht.end(); } - const_iterator begin() const { return _ht.begin(); } - const_iterator end() const { return _ht.end(); } - size_t capacity() const { return _ht.capacity(); } - size_t size() const { return _ht.size(); } - bool empty() const { return _ht.empty(); } + constexpr iterator begin() noexcept { return _ht.begin(); } + constexpr iterator end() noexcept { return _ht.end(); } + constexpr const_iterator begin() const noexcept { return _ht.begin(); } + constexpr const_iterator end() const noexcept { return _ht.end(); } + constexpr size_t capacity() const noexcept { return _ht.capacity(); } + constexpr size_t size() const noexcept { return _ht.size(); } + constexpr bool empty() const noexcept { return _ht.empty(); } insert_result insert(const value_type & value) { return _ht.insert(value); } insert_result insert(value_type &&value) { return _ht.insert(std::move(value)); } template <typename InputIt> @@ -51,17 +51,17 @@ public: void erase(const K & key); void erase(iterator it) { return erase(it->first); } void erase(const_iterator it) { return erase(it->first); } - iterator find(const K & key) { return _ht.find(key); } - size_t count(const K & key) const { return _ht.find(key) != _ht.end() ? 1 : 0; } - bool contains(const K & key) const { return _ht.find(key) != end(); } - const_iterator find(const K & key) const { return _ht.find(key); } + constexpr iterator find(const K & key) noexcept { return _ht.find(key); } + constexpr size_t count(const K & key) const noexcept { return _ht.find(key) != _ht.end() ? 1 : 0; } + constexpr bool contains(const K & key) const noexcept { return _ht.find(key) != end(); } + const_iterator find(const K & key) const noexcept { return _ht.find(key); } template< typename AltKey > const_iterator find(const AltKey & key) const { return _ht.template find<AltKey>(key); } template< typename AltKey> - iterator find(const AltKey & key) { + iterator find(const AltKey & key) noexcept { return _ht.template find<AltKey>(key); } diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.cpp b/vespalib/src/vespa/vespalib/stllike/hashtable.cpp index 9f0d0137e66..9ec66618ca5 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.cpp +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.cpp @@ -19,7 +19,7 @@ static const unsigned long __stl_prime_list[] = namespace vespalib { size_t -hashtable_base::getModulo(size_t newSize, const unsigned long * list, size_t sz) +hashtable_base::getModulo(size_t newSize, const unsigned long * list, size_t sz) noexcept { const unsigned long* first = list; const unsigned long* last = list + sz; @@ -28,7 +28,7 @@ hashtable_base::getModulo(size_t newSize, const unsigned long * list, size_t sz) } size_t -hashtable_base::getModuloStl(size_t size) +hashtable_base::getModuloStl(size_t size) noexcept { return getModulo(size, __stl_prime_list, sizeof(__stl_prime_list)/sizeof(__stl_prime_list[0])); } diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.h b/vespalib/src/vespa/vespalib/stllike/hashtable.h index 2ae63ae41f3..69b5aba57f5 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.h +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.h @@ -62,9 +62,9 @@ public: class prime_modulator { public: - prime_modulator(next_t sizeOfHashTable) : _modulo(sizeOfHashTable) { } - next_t modulo(next_t hash) const { return hash % _modulo; } - next_t getTableSize() const { return _modulo; } + prime_modulator(next_t sizeOfHashTable) noexcept : _modulo(sizeOfHashTable) { } + next_t modulo(next_t hash) const noexcept { return hash % _modulo; } + next_t getTableSize() const noexcept { return _modulo; } static next_t selectHashTableSize(size_t sz) { return hashtable_base::getModuloStl(sz); } private: next_t _modulo; @@ -76,15 +76,15 @@ public: class and_modulator { public: - and_modulator(next_t sizeOfHashTable) : _mask(sizeOfHashTable-1) { } - next_t modulo(next_t hash) const { return hash & _mask; } - next_t getTableSize() const { return _mask + 1; } - static next_t selectHashTableSize(size_t sz) { return hashtable_base::getModuloSimple(sz); } + and_modulator(next_t sizeOfHashTable) noexcept : _mask(sizeOfHashTable-1) { } + next_t modulo(next_t hash) const noexcept { return hash & _mask; } + next_t getTableSize() const noexcept { return _mask + 1; } + static next_t selectHashTableSize(size_t sz) noexcept { return hashtable_base::getModuloSimple(sz); } private: next_t _mask; }; - static size_t getModuloStl(size_t size); - static size_t getModuloSimple(size_t size) { + static size_t getModuloStl(size_t size) noexcept; + static size_t getModuloSimple(size_t size) noexcept { return std::max(size_t(8), roundUp2inN(size)); } protected: @@ -96,7 +96,7 @@ protected: } }; private: - static size_t getModulo(size_t newSize, const unsigned long * list, size_t sz); + static size_t getModulo(size_t newSize, const unsigned long * list, size_t sz) noexcept; }; template<typename V> @@ -104,20 +104,20 @@ class hash_node { public: using next_t=hashtable_base::next_t; enum {npos=-1u, invalid=-2u}; - hash_node() noexcept + constexpr hash_node() noexcept : _next(invalid) {} - hash_node(const V & node, next_t next=npos) noexcept(std::is_nothrow_copy_constructible_v<V>) + constexpr hash_node(const V & node, next_t next=npos) noexcept(std::is_nothrow_copy_constructible_v<V>) : _next(next) { new (_node) V(node); } - hash_node(V &&node, next_t next=npos) noexcept + constexpr hash_node(V &&node, next_t next=npos) noexcept : _next(next) { new (_node) V(std::move(node)); } - hash_node(hash_node && rhs) noexcept + constexpr hash_node(hash_node && rhs) noexcept : _next(rhs._next) { if (rhs.valid()) { @@ -134,7 +134,7 @@ public: } return *this; } - hash_node(const hash_node & rhs) noexcept(std::is_nothrow_copy_constructible_v<V>) + constexpr hash_node(const hash_node & rhs) noexcept(std::is_nothrow_copy_constructible_v<V>) : _next(rhs._next) { if (rhs.valid()) { @@ -154,20 +154,20 @@ public: ~hash_node() noexcept { destruct(); } - bool operator == (const hash_node & rhs) const noexcept { + constexpr bool operator == (const hash_node & rhs) const noexcept { return (_next == rhs._next) && (!valid() || (getValue() == rhs.getValue())); } V & getValue() noexcept { return *reinterpret_cast<V *>(_node); } - const V & getValue() const noexcept { return *reinterpret_cast<const V *>(_node); } - next_t getNext() const noexcept { return _next; } + constexpr const V & getValue() const noexcept { return *reinterpret_cast<const V *>(_node); } + constexpr next_t getNext() const noexcept { return _next; } void setNext(next_t next) noexcept { _next = next; } void invalidate() noexcept { destruct(); _next = invalid; } void terminate() noexcept { _next = npos; } - bool valid() const noexcept { return _next != invalid; } - bool hasNext() const noexcept { return valid() && (_next != npos); } + constexpr bool valid() const noexcept { return _next != invalid; } + constexpr bool hasNext() const noexcept { return valid() && (_next != npos); } private: void destruct() noexcept { if constexpr (!can_skip_destruction<V>) { @@ -198,32 +198,32 @@ public: using pointer = Value*; using iterator_category = std::forward_iterator_tag; - iterator(hashtable * hash) : _current(0), _hashTable(hash) { + constexpr iterator(hashtable * hash) noexcept : _current(0), _hashTable(hash) { if (! _hashTable->_nodes[_current].valid()) { advanceToNextValidHash(); } } - iterator(hashtable * hash, next_t pos) : _current(pos), _hashTable(hash) { } - static iterator end(hashtable *hash) { return iterator(hash, hash->initializedSize()); } + constexpr iterator(hashtable * hash, next_t pos) noexcept : _current(pos), _hashTable(hash) { } + constexpr static iterator end(hashtable *hash) noexcept { return iterator(hash, hash->initializedSize()); } - Value & operator * () const { return _hashTable->get(_current); } - Value * operator -> () const { return & _hashTable->get(_current); } - iterator & operator ++ () { + constexpr Value & operator * () const noexcept { return _hashTable->get(_current); } + constexpr Value * operator -> () const noexcept { return & _hashTable->get(_current); } + iterator & operator ++ () noexcept { advanceToNextValidHash(); return *this; } - iterator operator ++ (int) { + iterator operator ++ (int) noexcept { iterator prev = *this; ++(*this); return prev; } - bool operator==(const iterator& rhs) const { return (_current == rhs._current); } - bool operator!=(const iterator& rhs) const { return (_current != rhs._current); } + constexpr bool operator==(const iterator& rhs) const noexcept { return (_current == rhs._current); } + constexpr bool operator!=(const iterator& rhs) const noexcept { return (_current != rhs._current); } /// Carefull about this one. Only used by lrucache. - next_t getInternalIndex() const { return _current; } - void setInternalIndex(next_t n) { _current = n; } + constexpr next_t getInternalIndex() const noexcept { return _current; } + void setInternalIndex(next_t n) noexcept { _current = n; } private: - void advanceToNextValidHash() { + void advanceToNextValidHash() noexcept { ++_current; while ((_current < _hashTable->initializedSize()) && ! _hashTable->_nodes[_current].valid()) { ++_current; @@ -242,31 +242,31 @@ public: using pointer = const Value*; using iterator_category = std::forward_iterator_tag; - const_iterator(const hashtable * hash) : _current(0), _hashTable(hash) { + constexpr const_iterator(const hashtable * hash) noexcept : _current(0), _hashTable(hash) { if (! _hashTable->_nodes[_current].valid()) { advanceToNextValidHash(); } } - const_iterator(const hashtable * hash, next_t pos) : _current(pos), _hashTable(hash) { } - const_iterator(const iterator &i) : _current(i._current), _hashTable(i._hashTable) {} - static const_iterator end(const hashtable *hash) { return const_iterator(hash, hash->initializedSize()); } + constexpr const_iterator(const hashtable * hash, next_t pos) noexcept : _current(pos), _hashTable(hash) { } + constexpr const_iterator(const iterator &i) noexcept : _current(i._current), _hashTable(i._hashTable) {} + static constexpr const_iterator end(const hashtable *hash) noexcept { return const_iterator(hash, hash->initializedSize()); } - const Value & operator * () const { return _hashTable->get(_current); } - const Value * operator -> () const { return & _hashTable->get(_current); } - const_iterator & operator ++ () { + constexpr const Value & operator * () const noexcept { return _hashTable->get(_current); } + constexpr const Value * operator -> () const noexcept { return & _hashTable->get(_current); } + const_iterator & operator ++ () noexcept { advanceToNextValidHash(); return *this; } - const_iterator operator ++ (int) { + const_iterator operator ++ (int) noexcept { const_iterator prev = *this; ++(*this); return prev; } - bool operator==(const const_iterator& rhs) const { return (_current == rhs._current); } - bool operator!=(const const_iterator& rhs) const { return (_current != rhs._current); } - next_t getInternalIndex() const { return _current; } + constexpr bool operator==(const const_iterator& rhs) const { return (_current == rhs._current); } + constexpr bool operator!=(const const_iterator& rhs) const { return (_current != rhs._current); } + constexpr next_t getInternalIndex() const { return _current; } private: - void advanceToNextValidHash() { + void advanceToNextValidHash() noexcept { ++_current; while ((_current < _hashTable->initializedSize()) && ! _hashTable->_nodes[_current].valid()) { ++_current; @@ -285,20 +285,20 @@ public: hashtable(size_t reservedSpace); hashtable(size_t reservedSpace, const Hash & hasher, const Equal & equal); virtual ~hashtable(); - iterator begin() { return iterator(this); } - iterator end() { return iterator::end(this); } - const_iterator begin() const { return const_iterator(this); } - const_iterator end() const { return const_iterator::end(this); } - size_t capacity() const { return _nodes.capacity(); } - size_t size() const { return _count; } - bool empty() const { return _count == 0; } + constexpr iterator begin() noexcept { return iterator(this); } + constexpr iterator end() noexcept { return iterator::end(this); } + constexpr const_iterator begin() const noexcept { return const_iterator(this); } + constexpr const_iterator end() const noexcept { return const_iterator::end(this); } + constexpr size_t capacity() const noexcept { return _nodes.capacity(); } + constexpr size_t size() const noexcept { return _count; } + constexpr bool empty() const noexcept { return _count == 0; } template< typename AltKey> - iterator find(const AltKey & key); - iterator find(const Key & key); + iterator find(const AltKey & key) noexcept; + iterator find(const Key & key) noexcept; template< typename AltKey> - const_iterator find(const AltKey & key) const; - const_iterator find(const Key & key) const; + const_iterator find(const AltKey & key) const noexcept; + const_iterator find(const Key & key) const noexcept; template <typename V> insert_result insert(V && node) { return insert_internal(std::forward<V>(node)); @@ -338,12 +338,12 @@ protected: insert_result insert_internal(V && node); /// These two methods are only for the ones that know what they are doing. /// valid input here are stuff returned from iterator.getInternalIndex. - Value & getByInternalIndex(size_t index) { return _nodes[index].getValue(); } - const Value & getByInternalIndex(size_t index) const { return _nodes[index].getValue(); } + Value & getByInternalIndex(size_t index) noexcept { return _nodes[index].getValue(); } + constexpr const Value & getByInternalIndex(size_t index) const noexcept { return _nodes[index].getValue(); } template <typename MoveHandler> void erase(MoveHandler & moveHandler, next_t h, const const_iterator & key); template<typename K> - next_t hash(const K & key) const { return modulator(_hasher(key)); } + constexpr next_t hash(const K & key) const noexcept { return modulator(_hasher(key)); } private: Modulator _modulator; size_t _count; @@ -351,11 +351,11 @@ private: Hash _hasher; Equal _equal; KeyExtract _keyExtractor; - Value & get(size_t index) { return _nodes[index].getValue(); } - const Value & get(size_t index) const { return _nodes[index].getValue(); } - next_t modulator(next_t key) const { return _modulator.modulo(key); } - next_t getTableSize() const { return _modulator.getTableSize(); } - size_t initializedSize() const { return _nodes.size(); } + Value & get(size_t index) noexcept { return _nodes[index].getValue(); } + constexpr const Value & get(size_t index) const noexcept { return _nodes[index].getValue(); } + constexpr next_t modulator(next_t key) const noexcept { return _modulator.modulo(key); } + constexpr next_t getTableSize() const noexcept { return _modulator.getTableSize(); } + constexpr size_t initializedSize() const noexcept { return _nodes.size(); } template <typename MoveHandler> void move(MoveHandler & moveHandler, next_t from, next_t to) { _nodes[to] = std::move(_nodes[from]); diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.hpp b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp index 772d1bbae9b..a2e17817d62 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp @@ -68,7 +68,7 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::~hashtable() = defaul template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::iterator -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) noexcept { next_t h = hash(key); if (__builtin_expect(_nodes[h].valid(), true)) { @@ -84,7 +84,7 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::const_iterator -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) const +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) const noexcept { next_t h = hash(key); if (__builtin_expect(_nodes[h].valid(), true)) { @@ -101,7 +101,7 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > template< typename AltKey> typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::iterator -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & key) +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & key) noexcept { next_t h = hash(key); if (__builtin_expect(_nodes[h].valid(), true)) { @@ -118,7 +118,7 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & k template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > template< typename AltKey> typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::const_iterator -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & key) const +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & key) const noexcept { next_t h = hash(key); if (__builtin_expect(_nodes[h].valid(), true)) { diff --git a/vespalib/src/vespa/vespalib/stllike/lrucache_map.h b/vespalib/src/vespa/vespalib/stllike/lrucache_map.h index ffb7a427d11..6758b62d2c2 100644 --- a/vespalib/src/vespa/vespalib/stllike/lrucache_map.h +++ b/vespalib/src/vespa/vespalib/stllike/lrucache_map.h @@ -11,8 +11,8 @@ namespace vespalib { struct LinkedValueBase { static const uint32_t npos = static_cast<uint32_t>(-1); - LinkedValueBase() : _prev(npos), _next(npos) { } - LinkedValueBase(uint32_t prev, uint32_t next) : _prev(prev), _next(next) { } + constexpr LinkedValueBase() noexcept : _prev(npos), _next(npos) { } + constexpr LinkedValueBase(uint32_t prev, uint32_t next) noexcept : _prev(prev), _next(next) { } uint32_t _prev; uint32_t _next; }; @@ -20,9 +20,9 @@ struct LinkedValueBase { template<typename V> struct LinkedValue : public LinkedValueBase { - LinkedValue() {} - LinkedValue(const V & v) : LinkedValueBase(), _value(v) { } - LinkedValue(V && v) : LinkedValueBase(), _value(std::move(v)) { } + constexpr LinkedValue() noexcept {} + constexpr LinkedValue(const V & v) noexcept : LinkedValueBase(), _value(v) { } + constexpr LinkedValue(V && v) noexcept : LinkedValueBase(), _value(std::move(v)) { } V _value; }; @@ -151,7 +151,7 @@ public: * Tell if an object with given key exists in the cache. * Does not alter the LRU list. */ - bool hasKey(const K & key) const { return HashTable::find(key) != HashTable::end(); } + bool hasKey(const K & key) const __attribute__((noinline)); /** * Called when an object is inserted, to see if the LRU should be removed. diff --git a/vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp b/vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp index ccaae52469e..ca1b075d68b 100644 --- a/vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp +++ b/vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp @@ -266,6 +266,12 @@ lrucache_map<P>::operator [] (const K & key) } template< typename P > +bool +lrucache_map<P>::hasKey(const K & key) const { + return HashTable::find(key) != HashTable::end(); +} + +template< typename P > typename P::Value * lrucache_map<P>::findAndRef(const K & key) { |