From 8d5731bb1e2f47ca0833897256fb3933ce891eae Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Wed, 4 Sep 2019 13:05:13 +0000 Subject: Simplify how dictionary is populated when loading enum store. --- .../searchlib/attribute/enum_store_dictionary.cpp | 9 ----- .../searchlib/attribute/enum_store_dictionary.h | 1 - .../src/vespa/searchlib/attribute/enumstore.h | 8 ++--- .../src/vespa/searchlib/attribute/enumstore.hpp | 42 +++++++++++++--------- .../src/vespa/searchlib/attribute/i_enum_store.h | 18 ---------- .../searchlib/attribute/i_enum_store_dictionary.h | 1 - .../vespalib/datastore/i_unique_store_dictionary.h | 4 ++- .../vespalib/datastore/unique_store_dictionary.h | 1 + .../vespalib/datastore/unique_store_dictionary.hpp | 11 ++++++ 9 files changed, 44 insertions(+), 51 deletions(-) diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index a080c91cb34..0cd30c2b5cb 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -61,15 +61,6 @@ EnumStoreDictionary::writeAllValues(BufferWriter& writer, } } -template -ssize_t -EnumStoreDictionary::deserialize(const void* src, - size_t available, - IndexVector& idx) -{ - return _enumStore.deserialize(src, available, idx, this->_dict); -} - template void EnumStoreDictionary::fixupRefCounts(const EnumVector& hist) diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h index 38a7d6b1985..6367a06e339 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h @@ -34,7 +34,6 @@ public: uint32_t getNumUniques() const override; void writeAllValues(BufferWriter& writer, btree::BTreeNode::Ref rootRef) const override; - ssize_t deserialize(const void* src, size_t available, IndexVector& idx) override; void fixupRefCounts(const EnumVector& hist) override; void removeUnusedEnums(const IndexSet& unused, diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 3a493b74421..bc627cefe00 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -110,6 +110,8 @@ private: return _store.get_allocator().get_wrapped(idx); } + ssize_t deserialize_internal(const void* src, size_t available, IndexVector& idx); + public: EnumStoreT(bool has_postings); virtual ~EnumStoreT(); @@ -135,11 +137,7 @@ public: void transferHoldLists(generation_t generation); void trimHoldLists(generation_t firstUsed); - ssize_t deserialize0(const void *src, size_t available, IndexVector &idx) override; - - ssize_t deserialize(const void *src, size_t available, IndexVector &idx) override { - return _dict.deserialize(src, available, idx); - } + ssize_t deserialize(const void* src, size_t available, IndexVector& idx) override; void fixupRefCounts(const EnumVector &hist) override { _dict.fixupRefCounts(hist); } void freezeTree() { _store.freeze(); } diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index ee08de11fce..18990ecb414 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -32,6 +32,27 @@ void EnumStoreT::freeUnusedEnum(Index idx, IndexSet& unused) } } +template +ssize_t +EnumStoreT::deserialize_internal(const void* src, + size_t available, + IndexVector& idx) +{ + size_t left = available; + const char* p = static_cast(src); + Index idx1; + while (left > 0) { + ssize_t sz = deserialize(p, left, idx1); + if (sz < 0) { + return sz; + } + p += sz; + left -= sz; + idx.push_back(idx1); + } + return available - left; +} + template EnumStoreT::EnumStoreT(bool has_postings) : _store(make_enum_store_dictionary(*this, has_postings, EntryType::hasFold() ? std::make_unique(*this) : std::unique_ptr())), @@ -66,26 +87,15 @@ EnumStoreT::trimHoldLists(generation_t firstUsed) _store.trimHoldLists(firstUsed); } - template ssize_t -EnumStoreT::deserialize0(const void* src, - size_t available, - IndexVector& idx) +EnumStoreT::deserialize(const void* src, size_t available, IndexVector& idx) { - size_t left = available; - const char* p = static_cast(src); - Index idx1; - while (left > 0) { - ssize_t sz = deserialize(p, left, idx1); - if (sz < 0) { - return sz; - } - p += sz; - left -= sz; - idx.push_back(idx1); + ssize_t sz = deserialize_internal(src, available, idx); + if (sz >= 0) { + _dict.build(idx); } - return available - left; + return sz; } template diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index 7e9128c94be..5f6069a08f4 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -51,7 +51,6 @@ public: virtual ~IEnumStore() = default; virtual void writeValues(BufferWriter& writer, const Index* idxs, size_t count) const = 0; - virtual ssize_t deserialize0(const void* src, size_t available, IndexVector& idx) = 0; virtual ssize_t deserialize(const void* src, size_t available, IndexVector& idx) = 0; virtual void fixupRefCount(Index idx, uint32_t refCount) = 0; virtual void fixupRefCounts(const EnumVector& histogram) = 0; @@ -76,23 +75,6 @@ public: return enumstore::EnumeratedPostingsLoader(*this); } - template - ssize_t deserialize(const void* src, - size_t available, - IndexVector& idx, - TreeT& tree) { - ssize_t sz(deserialize0(src, available, idx)); - if (sz >= 0) { - typename TreeT::Builder builder(tree.getAllocator()); - typedef IndexVector::const_iterator IT; - for (IT i(idx.begin()), ie(idx.end()); i != ie; ++i) { - builder.insert(*i, typename TreeT::DataType()); - } - tree.assign(builder); - } - return sz; - } - template void fixupRefCounts(const EnumVector& hist, TreeT& tree) { if (hist.empty()) { diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h index 5c2510346c8..c167fabfdcd 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h @@ -39,7 +39,6 @@ public: virtual uint32_t getNumUniques() const = 0; virtual void writeAllValues(BufferWriter& writer, btree::BTreeNode::Ref rootRef) const = 0; - virtual ssize_t deserialize(const void* src, size_t available, IndexVector& idx) = 0; virtual void fixupRefCounts(const EnumVector& hist) = 0; virtual void freeUnusedEnums(const datastore::EntryComparator& cmp) = 0; diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h index a780cb4fe98..25480176143 100644 --- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h @@ -2,8 +2,9 @@ #pragma once -#include #include "entryref.h" +#include +#include #include namespace search::datastore { @@ -45,6 +46,7 @@ public: virtual uint32_t get_num_uniques() const = 0; virtual vespalib::MemoryUsage get_memory_usage() const = 0; virtual void build(const std::vector &refs, const std::vector &ref_counts, std::function hold) = 0; + virtual void build(const vespalib::ConstArrayRef& refs) = 0; virtual void build_with_payload(const std::vector& refs, const std::vector& payloads) = 0; virtual std::unique_ptr get_read_snapshot() const = 0; virtual EntryRef get_frozen_root() const = 0; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h index 15b947e283b..d5150aae236 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h @@ -48,6 +48,7 @@ public: uint32_t get_num_uniques() const override; vespalib::MemoryUsage get_memory_usage() const override; void build(const std::vector &refs, const std::vector &ref_counts, std::function hold) override; + void build(const vespalib::ConstArrayRef& refs) override; void build_with_payload(const std::vector& refs, const std::vector& payloads) override; std::unique_ptr get_read_snapshot() const override; EntryRef get_frozen_root() const override; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp index 3784b903ad6..896970ee3f1 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -175,6 +175,17 @@ UniqueStoreDictionary::build(const std::vector & _dict.assign(builder); } +template +void +UniqueStoreDictionary::build(const vespalib::ConstArrayRef& refs) +{ + typename DictionaryType::Builder builder(_dict.getAllocator()); + for (const auto& ref : refs) { + builder.insert(ref, DataType()); + } + _dict.assign(builder); +} + template void UniqueStoreDictionary::build_with_payload(const std::vector& refs, -- cgit v1.2.3