Improve memory management in all enum attributes.

The new enum store uses 1024 small data buffers instead of 2 large as before.
This avoids the problem with memory spikes when the active buffer was full and
all values had to be compacted into the other buffer.
In addition the new enum store uses free lists such that compaction is not needed as often.

7 files changed, 42 insertions, 1 deletions

diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
index cda62884318..a780cb4fe98 100644
--- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
+++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h
@@ -45,6 +45,7 @@ public:
     virtual uint32_t get_num_uniques() const = 0;
     virtual vespalib::MemoryUsage get_memory_usage() const = 0;
     virtual void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) = 0;
+    virtual void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) = 0;
     virtual std::unique_ptr<ReadSnapshot> get_read_snapshot() const = 0;
     virtual EntryRef get_frozen_root() const = 0;
 };
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.h b/vespalib/src/vespa/vespalib/datastore/unique_store.h
index bf7808e9325..6b85e79d3eb 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store.h
@@ -44,6 +44,7 @@ private:
 
 public:
     UniqueStore();
+    UniqueStore(std::unique_ptr<IUniqueStoreDictionary> dict);
     ~UniqueStore();
     UniqueStoreAddResult add(EntryConstRefType value);
     EntryRef find(EntryConstRefType value);
@@ -51,6 +52,12 @@ public:
     void remove(EntryRef ref);
     ICompactionContext::UP compactWorst();
     vespalib::MemoryUsage getMemoryUsage() const;
+    vespalib::AddressSpace get_address_space_usage() const;
+
+    // TODO: Consider exposing only the needed functions from allocator
+    Allocator& get_allocator() { return _allocator; }
+    const Allocator& get_allocator() const { return _allocator; }
+    IUniqueStoreDictionary& get_dictionary() { return *_dict; }
 
     // Pass on hold list management to underlying store
     void transferHoldLists(generation_t generation);
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
index f1b60845403..ebd81010612 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
@@ -28,9 +28,15 @@ using DefaultUniqueStoreDictionary = UniqueStoreDictionary<DefaultDictionary>;
 
 template <typename EntryT, typename RefT, typename Compare, typename Allocator>
 UniqueStore<EntryT, RefT, Compare, Allocator>::UniqueStore()
+    : UniqueStore<EntryT, RefT, Compare, Allocator>(std::make_unique<uniquestore::DefaultUniqueStoreDictionary>())
+{
+}
+
+template <typename EntryT, typename RefT, typename Compare, typename Allocator>
+UniqueStore<EntryT, RefT, Compare, Allocator>::UniqueStore(std::unique_ptr<IUniqueStoreDictionary> dict)
     : _allocator(),
       _store(_allocator.get_data_store()),
-      _dict(std::make_unique<uniquestore::DefaultUniqueStoreDictionary>())
+      _dict(std::move(dict))
 {
 }
 
@@ -178,6 +184,13 @@ UniqueStore<EntryT, RefT, Compare, Allocator>::getMemoryUsage() const
 }
 
 template <typename EntryT, typename RefT, typename Compare, typename Allocator>
+vespalib::AddressSpace
+UniqueStore<EntryT, RefT, Compare, Allocator>::get_address_space_usage() const
+{
+    return _allocator.get_data_store().getAddressSpaceUsage();
+}
+
+template <typename EntryT, typename RefT, typename Compare, typename Allocator>
 const BufferState &
 UniqueStore<EntryT, RefT, Compare, Allocator>::bufferState(EntryRef ref) const
 {
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.h
index 1981a190cc6..a4443742e33 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.h
@@ -42,6 +42,7 @@ public:
         return get_wrapped(ref).value();
     }
     DataStoreType& get_data_store() { return _store; }
+    const DataStoreType& get_data_store() const { return _store; }
 };
 
 }
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h
index a0e9f3d63a7..7f5162d97ff 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h
@@ -23,6 +23,7 @@ class UniqueStoreBuilder {
     IUniqueStoreDictionary& _dict;
     std::vector<EntryRef> _refs;
     std::vector<uint32_t> _refCounts;
+
 public:
     UniqueStoreBuilder(Allocator& allocator, IUniqueStoreDictionary& dict, uint32_t uniqueValuesHint);
     ~UniqueStoreBuilder();
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
index 4ae32c45dea..15b947e283b 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h
@@ -48,6 +48,7 @@ public:
     uint32_t get_num_uniques() const override;
     vespalib::MemoryUsage get_memory_usage() const override;
     void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) override;
+    void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) override;
     std::unique_ptr<ReadSnapshot> get_read_snapshot() const override;
     EntryRef get_frozen_root() const override;
 };
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
index f3087bc5610..3784b903ad6 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp
@@ -176,6 +176,23 @@ UniqueStoreDictionary<DictionaryT, ParentT>::build(const std::vector<EntryRef> &
 }
 
 template <typename DictionaryT, typename ParentT>
+void
+UniqueStoreDictionary<DictionaryT, ParentT>::build_with_payload(const std::vector<EntryRef>& refs,
+                                                                const std::vector<uint32_t>& payloads)
+{
+    assert(refs.size() == payloads.size());
+    typename DictionaryType::Builder builder(_dict.getAllocator());
+    for (size_t i = 0; i < refs.size(); ++i) {
+        if constexpr (std::is_same_v<DataType, uint32_t>) {
+            builder.insert(refs[i], payloads[i]);
+        } else {
+            builder.insert(refs[i], DataType());
+        }
+    }
+    _dict.assign(builder);
+}
+
+template <typename DictionaryT, typename ParentT>
 std::unique_ptr<typename ParentT::ReadSnapshot>
 UniqueStoreDictionary<DictionaryT, ParentT>::get_read_snapshot() const
 {