diff options
author | Geir Storli <geirst@verizonmedia.com> | 2019-09-10 13:36:55 +0000 |
---|---|---|
committer | Geir Storli <geirst@verizonmedia.com> | 2019-09-10 13:36:55 +0000 |
commit | 58bd3fe4539fcf80de794bc84c60c7c387637a42 (patch) | |
tree | f8372d93a39bd65b96efdd810f4e000240e3b7ce /searchlib/src | |
parent | 5608083ccf31ac994fe0ba090b9325ffcaab74b6 (diff) |
Simplify how we iterate the set of unique values when saving an enumerated attribute.
Diffstat (limited to 'searchlib/src')
8 files changed, 13 insertions, 44 deletions
diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index d45d6fa8c7c..dd58909ac7b 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -41,28 +41,6 @@ EnumStoreDictionary<DictionaryT>::getNumUniques() const template <typename DictionaryT> void -EnumStoreDictionary<DictionaryT>::writeAllValues(BufferWriter& writer, - BTreeNode::Ref rootRef) const -{ - constexpr size_t BATCHSIZE = 1000; - std::vector<Index> idxs; - idxs.reserve(BATCHSIZE); - typename DictionaryT::Iterator it(rootRef, this->_dict.getAllocator()); - while (it.valid()) { - if (idxs.size() >= idxs.capacity()) { - _enumStore.writeValues(writer, vespalib::ConstArrayRef(&idxs[0], idxs.size())); - idxs.clear(); - } - idxs.push_back(it.getKey()); - ++it; - } - if (!idxs.empty()) { - _enumStore.writeValues(writer, vespalib::ConstArrayRef(&idxs[0], idxs.size())); - } -} - -template <typename DictionaryT> -void EnumStoreDictionary<DictionaryT>::fixupRefCounts(const EnumVector& hist) { _enumStore.fixupRefCounts(hist, this->_dict); diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h index 6367a06e339..7c33375bb21 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h @@ -33,7 +33,6 @@ public: DictionaryT &getDictionary() { return this->_dict; } uint32_t getNumUniques() const override; - void writeAllValues(BufferWriter& writer, btree::BTreeNode::Ref rootRef) const override; void fixupRefCounts(const EnumVector& hist) override; void removeUnusedEnums(const IndexSet& unused, diff --git a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp index ae6fa6580c1..cf394623abd 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp @@ -23,10 +23,10 @@ void EnumAttributeSaver::writeUdat(IAttributeSaveTarget &saveTarget) { if (saveTarget.getEnumerated()) { - std::unique_ptr<BufferWriter> - udatWriter(saveTarget.udatWriter().allocBufferWriter()); - const auto& enumDict = _enumStore.getEnumStoreDict(); - enumDict.writeAllValues(*udatWriter, _enumerator->get_frozen_root()); + auto udatWriter = saveTarget.udatWriter().allocBufferWriter(); + _enumerator->foreach_key([&](datastore::EntryRef idx){ + _enumStore.write_value(*udatWriter, idx); + }); udatWriter->flush(); } } diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp index 188a381682f..f47d81a15a4 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp @@ -11,14 +11,11 @@ namespace search { template <> void -EnumStoreT<const char*>::writeValues(BufferWriter& writer, - vespalib::ConstArrayRef<Index> idxs) const +EnumStoreT<const char*>::write_value(BufferWriter& writer, Index idx) const { - for (const auto& idx : idxs) { - const char* src = _store.get(idx); - size_t sz = strlen(src) + 1; - writer.write(src, sz); - } + const char* src = _store.get(idx); + size_t sz = strlen(src) + 1; + writer.write(src, sz); } template <> diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index bb07eaecd5e..399956db2fc 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -196,8 +196,7 @@ public: return FoldedComparatorType(_store.get_data_store(), fallback_value, prefix); } - // TODO: Change to sending enum indexes as const array ref. - void writeValues(BufferWriter& writer, vespalib::ConstArrayRef<Index> idxs) const override; + void write_value(BufferWriter& writer, Index idx) const override; bool foldedChange(const Index &idx1, const Index &idx2) const override; bool findEnum(EntryType value, IEnumStore::EnumHandle &e) const; std::vector<IEnumStore::EnumHandle> findFoldedEnums(EntryType value) const; @@ -227,8 +226,7 @@ class datastore::DataStoreT<IEnumStore::Index>; template <> void -EnumStoreT<const char*>::writeValues(BufferWriter& writer, - vespalib::ConstArrayRef<IEnumStore::Index> idxs) const; +EnumStoreT<const char*>::write_value(BufferWriter& writer, Index idx) const; template <> ssize_t diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 0beee6e0821..e2d07e6424d 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -148,11 +148,9 @@ EnumStoreT<EntryType>::BatchUpdater::insert(EntryType value) template <class EntryType> void -EnumStoreT<EntryType>::writeValues(BufferWriter& writer, vespalib::ConstArrayRef<Index> idxs) const +EnumStoreT<EntryType>::write_value(BufferWriter& writer, Index idx) const { - for (const auto& idx : idxs) { - writer.write(&_store.get(idx), sizeof(EntryType)); - } + writer.write(&_store.get(idx), sizeof(EntryType)); } template <class EntryType> diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index e55d41f12ab..a5a888f10e7 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -52,7 +52,7 @@ public: virtual ~IEnumStore() = default; - virtual void writeValues(BufferWriter& writer, vespalib::ConstArrayRef<Index> idxs) const = 0; + virtual void write_value(BufferWriter& writer, Index idx) const = 0; virtual ssize_t load_unique_values(const void* src, size_t available, IndexVector& idx) = 0; virtual void fixupRefCount(Index idx, uint32_t refCount) = 0; virtual void fixupRefCounts(const EnumVector& histogram) = 0; diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h index c167fabfdcd..717a365d4c9 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h @@ -38,7 +38,6 @@ public: virtual ~IEnumStoreDictionary() = default; virtual uint32_t getNumUniques() const = 0; - virtual void writeAllValues(BufferWriter& writer, btree::BTreeNode::Ref rootRef) const = 0; virtual void fixupRefCounts(const EnumVector& hist) = 0; virtual void freeUnusedEnums(const datastore::EntryComparator& cmp) = 0; |