diff options
author | Tor Egge <tegge@oath.com> | 2017-08-16 11:39:02 +0000 |
---|---|---|
committer | Tor Egge <tegge@oath.com> | 2017-08-16 11:39:02 +0000 |
commit | acfef4e09234fddf4b1cb57d7458352b0e867ad4 (patch) | |
tree | bb5e8d77a25a682f242bc32f2b4dcf34d26d5ae4 | |
parent | 78aa7deb15982f14ce56c88c329ee6f547345368 (diff) |
Maintain direct forward mapping from lid to referenced lid in reference
mapping.
6 files changed, 74 insertions, 19 deletions
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index 3cef9a03c8f..c431b956c1d 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -18,7 +18,8 @@ ImportedSearchContext::ImportedSearchContext( : _imported_attribute(imported_attribute), _reference_attribute(*_imported_attribute.getReferenceAttribute()), _target_attribute(*_imported_attribute.getTargetAttribute()), - _target_search_context(_target_attribute.getSearch(std::move(term), params)) + _target_search_context(_target_attribute.getSearch(std::move(term), params)), + _referencedLids(_reference_attribute.getReferencedLids()) { } @@ -60,11 +61,11 @@ const vespalib::string& ImportedSearchContext::attributeName() const { } bool ImportedSearchContext::cmp(DocId docId, int32_t& weight) const { - return _target_search_context->cmp(_reference_attribute.getReferencedLid(docId), weight); + return _target_search_context->cmp(_referencedLids[docId], weight); } bool ImportedSearchContext::cmp(DocId docId) const { - return _target_search_context->cmp(_reference_attribute.getReferencedLid(docId)); + return _target_search_context->cmp(_referencedLids[docId]); } } // attribute diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index 3863297afb4..ce6642cd93f 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -4,6 +4,7 @@ #include "attributevector.h" #include <vespa/searchcommon/attribute/i_search_context.h> +#include <vespa/vespalib/util/arrayref.h> #include <memory> namespace search { @@ -27,10 +28,12 @@ class SearchContextParams; * considered a match. */ class ImportedSearchContext : public ISearchContext { + using ReferencedLids = vespalib::ConstArrayRef<uint32_t>; const ImportedAttributeVector& _imported_attribute; const ReferenceAttribute& _reference_attribute; const AttributeVector& _target_attribute; std::unique_ptr<AttributeVector::SearchContext> _target_search_context; + ReferencedLids _referencedLids; public: ImportedSearchContext(std::unique_ptr<QueryTermSimple> term, const SearchContextParams& params, diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index ee911c9a8cc..487037a881b 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -59,6 +59,7 @@ void ReferenceAttribute::onAddDocs(DocId limit) { _indices.reserve(limit); + _referenceMappings.onAddDocs(limit); } bool @@ -67,6 +68,7 @@ ReferenceAttribute::addDoc(DocId &doc) bool incGen = _indices.isFull(); doc = _indices.size(); _indices.push_back(EntryRef()); + _referenceMappings.addDoc(); incNumDocs(); updateUncommittedDocIdLimit(doc); if (incGen) { @@ -222,6 +224,7 @@ ReferenceAttribute::onLoad() builder.add(value); } builder.setupRefCounts(); + _referenceMappings.onLoad(numDocs); _indices.clear(); _indices.unsafe_reserve(numDocs); for (uint32_t doc = 0; doc < numDocs; ++doc) { @@ -315,18 +318,6 @@ ReferenceAttribute::setGidToLidMapperFactory(std::shared_ptr<IGidToLidMapperFact _gidToLidMapperFactory = std::move(gidToLidMapperFactory); } -ReferenceAttribute::DocId -ReferenceAttribute::getReferencedLid(DocId doc) const -{ - assert(doc < _indices.size()); - EntryRef ref = _indices[doc]; - if (!ref.valid()) { - return 0; - } else { - return _store.get(ref).lid(); - } -} - void ReferenceAttribute::notifyGidToLidChange(const GlobalId &gid, DocId referencedLid) { @@ -349,7 +340,9 @@ ReferenceAttribute::populateReferencedLids() saver.foreach_key([&store,&mapper,this](EntryRef ref) { const Reference &entry = store.get(ref); entry.setLid(mapper.mapGidToLid(entry.gid())); - _referenceMappings.syncReverseMappingIndices(entry); }); + _referenceMappings.syncReverseMappingIndices(entry); + _referenceMappings.syncForwardMapping(entry); + }); } commit(); } @@ -376,6 +369,7 @@ ReferenceAttribute::onShrinkLidSpace() uint32_t committedDocIdLimit = getCommittedDocIdLimit(); assert(_indices.size() >= committedDocIdLimit); _indices.shrink(committedDocIdLimit); + _referenceMappings.shrink(committedDocIdLimit); setNumDocs(committedDocIdLimit); } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h index a751213dc9a..08fc384338c 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h @@ -37,6 +37,7 @@ public: std::less<uint32_t>, btree::BTreeDefaultTraits, btree::NoAggrCalc>; + using ReferencedLids = ReferenceMappings::ReferencedLids; private: ReferenceStore _store; ReferenceStoreIndices _indices; @@ -73,7 +74,8 @@ public: const Reference *getReference(DocId doc); void setGidToLidMapperFactory(std::shared_ptr<IGidToLidMapperFactory> gidToLidMapperFactory); std::shared_ptr<IGidToLidMapperFactory> getGidToLidMapperFactory() const { return _gidToLidMapperFactory; } - DocId getReferencedLid(DocId doc) const; + ReferencedLids getReferencedLids() const { return _referenceMappings.getReferencedLids(); } + DocId getReferencedLid(DocId doc) const { return _referenceMappings.getReferencedLid(doc); } void notifyGidToLidChange(const GlobalId &gid, DocId referencedLid); void populateReferencedLids(); virtual void clearDocs(DocId lidLow, DocId lidLimit) override; diff --git a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp index 67f02c50d88..f699b2e2c07 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp @@ -9,7 +9,8 @@ namespace search::attribute { ReferenceMappings::ReferenceMappings(GenerationHolder &genHolder) : _reverseMappingIndices(genHolder), - _reverseMapping() + _reverseMapping(), + _referencedLids(genHolder) { } @@ -27,6 +28,18 @@ ReferenceMappings::clearMapping(const Reference &entry) } void +ReferenceMappings::syncForwardMapping(const Reference &entry) +{ + uint32_t referencedLid = entry.lid(); + EntryRef revMapIdx = entry.revMapIdx(); + auto &referencedLids = _referencedLids; + _reverseMapping.foreach_unfrozen_key(revMapIdx, + [&referencedLids, referencedLid](uint32_t lid) + { referencedLids[lid] = referencedLid; }); +} + + +void ReferenceMappings::syncReverseMappingIndices(const Reference &entry) { uint32_t referencedLid = entry.lid(); @@ -44,6 +57,7 @@ ReferenceMappings::removeReverseMapping(const Reference &entry, uint32_t lid) std::atomic_thread_fence(std::memory_order_release); entry.setRevMapIdx(revMapIdx); syncReverseMappingIndices(entry); + _referencedLids[lid] = 0; // forward mapping } void @@ -55,6 +69,7 @@ ReferenceMappings::addReverseMapping(const Reference &entry, uint32_t lid) std::atomic_thread_fence(std::memory_order_release); entry.setRevMapIdx(revMapIdx); syncReverseMappingIndices(entry); + _referencedLids[lid] = entry.lid(); // forward mapping } void @@ -77,6 +92,32 @@ ReferenceMappings::notifyGidToLidChange(const Reference &entry, uint32_t referen entry.setLid(referencedLid); } syncReverseMappingIndices(entry); + syncForwardMapping(entry); +} + +void +ReferenceMappings::onAddDocs(uint32_t limit) +{ + _referencedLids.reserve(limit); +} + +void +ReferenceMappings::addDoc() +{ + _referencedLids.push_back(0); +} + +void +ReferenceMappings::onLoad(uint32_t numDocs) +{ + _referencedLids.clear(); + _referencedLids.unsafe_reserve(numDocs); +} + +void +ReferenceMappings::shrink(uint32_t limit) +{ + _referencedLids.shrink(limit); } } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_mappings.h b/searchlib/src/vespa/searchlib/attribute/reference_mappings.h index 129ccbbea25..925aae25719 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_mappings.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_mappings.h @@ -30,9 +30,12 @@ class ReferenceMappings // Store of B-Trees, used to map from gid or referenced lid to // referencing lids. ReverseMapping _reverseMapping; - + // vector containing referenced lid given referencing lid + RcuVectorBase<uint32_t> _referencedLids; public: + using ReferencedLids = vespalib::ConstArrayRef<uint32_t>; + ReferenceMappings(GenerationHolder &genHolder); ~ReferenceMappings(); @@ -52,6 +55,14 @@ public: void removeReverseMapping(const Reference &entry, uint32_t lid); void addReverseMapping(const Reference &entry, uint32_t lid); + // Maintain size of mapping from lid to referenced lid + void onAddDocs(uint32_t limit); + void addDoc(); + void onLoad(uint32_t numDocs); + void shrink(uint32_t limit); + + void syncForwardMapping(const Reference &entry); + // Setup mapping after load void buildReverseMapping(const Reference &entry, const std::vector<ReverseMapping::KeyDataType> &adds); @@ -59,6 +70,9 @@ public: template <typename FunctionType> void foreach_lid(uint32_t referencedLid, FunctionType &&func) const; + + ReferencedLids getReferencedLids() const { return ReferencedLids(&_referencedLids[0], _referencedLids.size()); } + uint32_t getReferencedLid(uint32_t doc) const { return _referencedLids[doc]; } }; template <typename FunctionType> |