Simplify how dictionary is populated when loading enum store.

9 files changed, 44 insertions, 51 deletions

diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp
index a080c91cb34..0cd30c2b5cb 100644
--- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp
@@ -62,15 +62,6 @@ EnumStoreDictionary<DictionaryT>::writeAllValues(BufferWriter& writer,
 }
 
 template <typename DictionaryT>
-ssize_t
-EnumStoreDictionary<DictionaryT>::deserialize(const void* src,
-                                              size_t available,
-                                              IndexVector& idx)
-{
-    return _enumStore.deserialize(src, available, idx, this->_dict);
-}
-
-template <typename DictionaryT>
 void
 EnumStoreDictionary<DictionaryT>::fixupRefCounts(const EnumVector& hist)
 {
diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h
index 38a7d6b1985..6367a06e339 100644
--- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h
+++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h
@@ -34,7 +34,6 @@ public:
 
     uint32_t getNumUniques() const override;
     void writeAllValues(BufferWriter& writer, btree::BTreeNode::Ref rootRef) const override;
-    ssize_t deserialize(const void* src, size_t available, IndexVector& idx) override;
     void fixupRefCounts(const EnumVector& hist) override;
 
     void removeUnusedEnums(const IndexSet& unused,
diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h
index 3a493b74421..bc627cefe00 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumstore.h
+++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h
@@ -110,6 +110,8 @@ private:
         return _store.get_allocator().get_wrapped(idx);
     }
 
+    ssize_t deserialize_internal(const void* src, size_t available, IndexVector& idx);
+
 public:
     EnumStoreT(bool has_postings);
     virtual ~EnumStoreT();
@@ -135,11 +137,7 @@ public:
     void transferHoldLists(generation_t generation);
     void trimHoldLists(generation_t firstUsed);
 
-    ssize_t deserialize0(const void *src, size_t available, IndexVector &idx) override;
-
-    ssize_t deserialize(const void *src, size_t available, IndexVector &idx) override {
-        return _dict.deserialize(src, available, idx);
-    }
+    ssize_t deserialize(const void* src, size_t available, IndexVector& idx) override;
 
     void fixupRefCounts(const EnumVector &hist) override { _dict.fixupRefCounts(hist); }
     void freezeTree() { _store.freeze(); }
diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
index ee08de11fce..18990ecb414 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
@@ -33,6 +33,27 @@ void EnumStoreT<EntryType>::freeUnusedEnum(Index idx, IndexSet& unused)
 }
 
 template <typename EntryType>
+ssize_t
+EnumStoreT<EntryType>::deserialize_internal(const void* src,
+                                            size_t available,
+                                            IndexVector& idx)
+{
+    size_t left = available;
+    const char* p = static_cast<const char*>(src);
+    Index idx1;
+    while (left > 0) {
+        ssize_t sz = deserialize(p, left, idx1);
+        if (sz < 0) {
+            return sz;
+        }
+        p += sz;
+        left -= sz;
+        idx.push_back(idx1);
+    }
+    return available - left;
+}
+
+template <typename EntryType>
 EnumStoreT<EntryType>::EnumStoreT(bool has_postings)
     : _store(make_enum_store_dictionary(*this, has_postings, EntryType::hasFold() ? std::make_unique<FoldedComparatorType>(*this) : std::unique_ptr<datastore::EntryComparator>())),
       _dict(static_cast<IEnumStoreDictionary&>(_store.get_dictionary())),
@@ -66,26 +87,15 @@ EnumStoreT<EntryType>::trimHoldLists(generation_t firstUsed)
     _store.trimHoldLists(firstUsed);
 }
 
-
 template <typename EntryType>
 ssize_t
-EnumStoreT<EntryType>::deserialize0(const void* src,
-                                    size_t available,
-                                    IndexVector& idx)
+EnumStoreT<EntryType>::deserialize(const void* src, size_t available, IndexVector& idx)
 {
-    size_t left = available;
-    const char* p = static_cast<const char*>(src);
-    Index idx1;
-    while (left > 0) {
-        ssize_t sz = deserialize(p, left, idx1);
-        if (sz < 0) {
-            return sz;
-        }
-        p += sz;
-        left -= sz;
-        idx.push_back(idx1);
+    ssize_t sz = deserialize_internal(src, available, idx);
+    if (sz >= 0) {
+        _dict.build(idx);
     }
-    return available - left;
+    return sz;
 }
 
 template <typename EntryType>
diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h
index 7e9128c94be..5f6069a08f4 100644
--- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h
+++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h
@@ -51,7 +51,6 @@ public:
     virtual ~IEnumStore() = default;
 
     virtual void writeValues(BufferWriter& writer, const Index* idxs, size_t count) const = 0;
-    virtual ssize_t deserialize0(const void* src, size_t available, IndexVector& idx) = 0;
     virtual ssize_t deserialize(const void* src, size_t available, IndexVector& idx) = 0;
     virtual void fixupRefCount(Index idx, uint32_t refCount) = 0;
     virtual void fixupRefCounts(const EnumVector& histogram) = 0;
@@ -77,23 +76,6 @@ public:
     }
 
     template <typename TreeT>
-    ssize_t deserialize(const void* src,
-                        size_t available,
-                        IndexVector& idx,
-                        TreeT& tree) {
-        ssize_t sz(deserialize0(src, available, idx));
-        if (sz >= 0) {
-            typename TreeT::Builder builder(tree.getAllocator());
-            typedef IndexVector::const_iterator IT;
-            for (IT i(idx.begin()), ie(idx.end()); i != ie; ++i) {
-                builder.insert(*i, typename TreeT::DataType());
-            }
-            tree.assign(builder);
-        }
-        return sz;
-    }
-
-    template <typename TreeT>
     void fixupRefCounts(const EnumVector& hist, TreeT& tree) {
         if (hist.empty()) {
             return;
diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h
index 5c2510346c8..c167fabfdcd 100644
--- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h
+++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h
@@ -39,7 +39,6 @@ public:
 
     virtual uint32_t getNumUniques() const = 0;
     virtual void writeAllValues(BufferWriter& writer, btree::BTreeNode::Ref rootRef) const = 0;
-    virtual ssize_t deserialize(const void* src, size_t available, IndexVector& idx) = 0;
 
     virtual void fixupRefCounts(const EnumVector& hist) = 0;
     virtual void freeUnusedEnums(const datastore::EntryComparator& cmp) = 0;
diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
index a780cb4fe98..25480176143 100644
--- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
+++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
@@ -2,8 +2,9 @@
 
 #pragma once
 
-#include <vespa/vespalib/util/generationhandler.h>
 #include "entryref.h"
+#include <vespa/vespalib/util/arrayref.h>
+#include <vespa/vespalib/util/generationhandler.h>
 #include <functional>
 
 namespace search::datastore {
@@ -45,6 +46,7 @@ public:
     virtual uint32_t get_num_uniques() const = 0;
     virtual vespalib::MemoryUsage get_memory_usage() const = 0;
     virtual void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) = 0;
+    virtual void build(const vespalib::ConstArrayRef<EntryRef>& refs) = 0;
     virtual void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) = 0;
     virtual std::unique_ptr<ReadSnapshot> get_read_snapshot() const = 0;
     virtual EntryRef get_frozen_root() const = 0;
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
index 15b947e283b..d5150aae236 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
@@ -48,6 +48,7 @@ public:
     uint32_t get_num_uniques() const override;
     vespalib::MemoryUsage get_memory_usage() const override;
     void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) override;
+    void build(const vespalib::ConstArrayRef<EntryRef>& refs) override;
     void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) override;
     std::unique_ptr<ReadSnapshot> get_read_snapshot() const override;
     EntryRef get_frozen_root() const override;
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
index 3784b903ad6..896970ee3f1 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
@@ -177,6 +177,17 @@ UniqueStoreDictionary<DictionaryT, ParentT>::build(const std::vector<EntryRef> &
 
 template <typename DictionaryT, typename ParentT>
 void
+UniqueStoreDictionary<DictionaryT, ParentT>::build(const vespalib::ConstArrayRef<EntryRef>& refs)
+{
+    typename DictionaryType::Builder builder(_dict.getAllocator());
+    for (const auto& ref : refs) {
+        builder.insert(ref, DataType());
+    }
+    _dict.assign(builder);
+}
+
+template <typename DictionaryT, typename ParentT>
+void
 UniqueStoreDictionary<DictionaryT, ParentT>::build_with_payload(const std::vector<EntryRef>& refs,
                                                                 const std::vector<uint32_t>& payloads)
 {