diff options
9 files changed, 44 insertions, 51 deletions
diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index a080c91cb34..0cd30c2b5cb 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -62,15 +62,6 @@ EnumStoreDictionary<DictionaryT>::writeAllValues(BufferWriter& writer, } template <typename DictionaryT> -ssize_t -EnumStoreDictionary<DictionaryT>::deserialize(const void* src, - size_t available, - IndexVector& idx) -{ - return _enumStore.deserialize(src, available, idx, this->_dict); -} - -template <typename DictionaryT> void EnumStoreDictionary<DictionaryT>::fixupRefCounts(const EnumVector& hist) { diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h index 38a7d6b1985..6367a06e339 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h @@ -34,7 +34,6 @@ public: uint32_t getNumUniques() const override; void writeAllValues(BufferWriter& writer, btree::BTreeNode::Ref rootRef) const override; - ssize_t deserialize(const void* src, size_t available, IndexVector& idx) override; void fixupRefCounts(const EnumVector& hist) override; void removeUnusedEnums(const IndexSet& unused, diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 3a493b74421..bc627cefe00 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -110,6 +110,8 @@ private: return _store.get_allocator().get_wrapped(idx); } + ssize_t deserialize_internal(const void* src, size_t available, IndexVector& idx); + public: EnumStoreT(bool has_postings); virtual ~EnumStoreT(); @@ -135,11 +137,7 @@ public: void transferHoldLists(generation_t generation); void trimHoldLists(generation_t firstUsed); - ssize_t deserialize0(const void *src, size_t available, IndexVector &idx) override; - - ssize_t deserialize(const void *src, size_t available, IndexVector &idx) override { - return _dict.deserialize(src, available, idx); - } + ssize_t deserialize(const void* src, size_t available, IndexVector& idx) override; void fixupRefCounts(const EnumVector &hist) override { _dict.fixupRefCounts(hist); } void freezeTree() { _store.freeze(); } diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index ee08de11fce..18990ecb414 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -33,6 +33,27 @@ void EnumStoreT<EntryType>::freeUnusedEnum(Index idx, IndexSet& unused) } template <typename EntryType> +ssize_t +EnumStoreT<EntryType>::deserialize_internal(const void* src, + size_t available, + IndexVector& idx) +{ + size_t left = available; + const char* p = static_cast<const char*>(src); + Index idx1; + while (left > 0) { + ssize_t sz = deserialize(p, left, idx1); + if (sz < 0) { + return sz; + } + p += sz; + left -= sz; + idx.push_back(idx1); + } + return available - left; +} + +template <typename EntryType> EnumStoreT<EntryType>::EnumStoreT(bool has_postings) : _store(make_enum_store_dictionary(*this, has_postings, EntryType::hasFold() ? std::make_unique<FoldedComparatorType>(*this) : std::unique_ptr<datastore::EntryComparator>())), _dict(static_cast<IEnumStoreDictionary&>(_store.get_dictionary())), @@ -66,26 +87,15 @@ EnumStoreT<EntryType>::trimHoldLists(generation_t firstUsed) _store.trimHoldLists(firstUsed); } - template <typename EntryType> ssize_t -EnumStoreT<EntryType>::deserialize0(const void* src, - size_t available, - IndexVector& idx) +EnumStoreT<EntryType>::deserialize(const void* src, size_t available, IndexVector& idx) { - size_t left = available; - const char* p = static_cast<const char*>(src); - Index idx1; - while (left > 0) { - ssize_t sz = deserialize(p, left, idx1); - if (sz < 0) { - return sz; - } - p += sz; - left -= sz; - idx.push_back(idx1); + ssize_t sz = deserialize_internal(src, available, idx); + if (sz >= 0) { + _dict.build(idx); } - return available - left; + return sz; } template <typename EntryType> diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index 7e9128c94be..5f6069a08f4 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -51,7 +51,6 @@ public: virtual ~IEnumStore() = default; virtual void writeValues(BufferWriter& writer, const Index* idxs, size_t count) const = 0; - virtual ssize_t deserialize0(const void* src, size_t available, IndexVector& idx) = 0; virtual ssize_t deserialize(const void* src, size_t available, IndexVector& idx) = 0; virtual void fixupRefCount(Index idx, uint32_t refCount) = 0; virtual void fixupRefCounts(const EnumVector& histogram) = 0; @@ -77,23 +76,6 @@ public: } template <typename TreeT> - ssize_t deserialize(const void* src, - size_t available, - IndexVector& idx, - TreeT& tree) { - ssize_t sz(deserialize0(src, available, idx)); - if (sz >= 0) { - typename TreeT::Builder builder(tree.getAllocator()); - typedef IndexVector::const_iterator IT; - for (IT i(idx.begin()), ie(idx.end()); i != ie; ++i) { - builder.insert(*i, typename TreeT::DataType()); - } - tree.assign(builder); - } - return sz; - } - - template <typename TreeT> void fixupRefCounts(const EnumVector& hist, TreeT& tree) { if (hist.empty()) { return; diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h index 5c2510346c8..c167fabfdcd 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h @@ -39,7 +39,6 @@ public: virtual uint32_t getNumUniques() const = 0; virtual void writeAllValues(BufferWriter& writer, btree::BTreeNode::Ref rootRef) const = 0; - virtual ssize_t deserialize(const void* src, size_t available, IndexVector& idx) = 0; virtual void fixupRefCounts(const EnumVector& hist) = 0; virtual void freeUnusedEnums(const datastore::EntryComparator& cmp) = 0; diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h index a780cb4fe98..25480176143 100644 --- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h @@ -2,8 +2,9 @@ #pragma once -#include <vespa/vespalib/util/generationhandler.h> #include "entryref.h" +#include <vespa/vespalib/util/arrayref.h> +#include <vespa/vespalib/util/generationhandler.h> #include <functional> namespace search::datastore { @@ -45,6 +46,7 @@ public: virtual uint32_t get_num_uniques() const = 0; virtual vespalib::MemoryUsage get_memory_usage() const = 0; virtual void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) = 0; + virtual void build(const vespalib::ConstArrayRef<EntryRef>& refs) = 0; virtual void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) = 0; virtual std::unique_ptr<ReadSnapshot> get_read_snapshot() const = 0; virtual EntryRef get_frozen_root() const = 0; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h index 15b947e283b..d5150aae236 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h @@ -48,6 +48,7 @@ public: uint32_t get_num_uniques() const override; vespalib::MemoryUsage get_memory_usage() const override; void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) override; + void build(const vespalib::ConstArrayRef<EntryRef>& refs) override; void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) override; std::unique_ptr<ReadSnapshot> get_read_snapshot() const override; EntryRef get_frozen_root() const override; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp index 3784b903ad6..896970ee3f1 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -177,6 +177,17 @@ UniqueStoreDictionary<DictionaryT, ParentT>::build(const std::vector<EntryRef> & template <typename DictionaryT, typename ParentT> void +UniqueStoreDictionary<DictionaryT, ParentT>::build(const vespalib::ConstArrayRef<EntryRef>& refs) +{ + typename DictionaryType::Builder builder(_dict.getAllocator()); + for (const auto& ref : refs) { + builder.insert(ref, DataType()); + } + _dict.assign(builder); +} + +template <typename DictionaryT, typename ParentT> +void UniqueStoreDictionary<DictionaryT, ParentT>::build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) { |