Merge pull request #26348 from vespa-engine/toregge/add-raw-buffer-store

Add RawBufferStore.

5 files changed, 117 insertions, 58 deletions

diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
index e3028773b75..663c760fc44 100644
--- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
@@ -104,6 +104,7 @@ vespa_add_library(searchlib_attribute OBJECT
     postinglisttraits.cpp
     postingstore.cpp
     predicate_attribute.cpp
+    raw_buffer_store.cpp
     raw_buffer_type_mapper.cpp
     raw_multi_value_read_view.cpp
     readerbase.cpp
diff --git a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp
new file mode 100644
index 00000000000..74894728ff4
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp
@@ -0,0 +1,69 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "raw_buffer_store.h"
+#include <vespa/vespalib/datastore/array_store.hpp>
+#include <cassert>
+
+using vespalib::alloc::MemoryAllocator;
+using vespalib::datastore::EntryRef;
+
+namespace {
+
+constexpr float ALLOC_GROW_FACTOR = 0.2;
+
+}
+
+namespace search::attribute {
+
+RawBufferStore::RawBufferStore(std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator, uint32_t max_small_buffer_type_id, double grow_factor)
+    : _array_store(ArrayStoreType::optimizedConfigForHugePage(max_small_buffer_type_id,
+                                                              RawBufferTypeMapper(max_small_buffer_type_id, grow_factor),
+                                                              MemoryAllocator::HUGEPAGE_SIZE,
+                                                              MemoryAllocator::PAGE_SIZE,
+                                                              8_Ki, ALLOC_GROW_FACTOR),
+                   std::move(allocator), RawBufferTypeMapper(max_small_buffer_type_id, grow_factor))
+{
+}
+
+RawBufferStore::~RawBufferStore() = default;
+
+vespalib::ConstArrayRef<char>
+RawBufferStore::get(EntryRef ref) const
+{
+    auto array = _array_store.get(ref);
+    uint32_t size = 0;
+    assert(array.size() >= sizeof(size));
+    memcpy(&size, array.data(), sizeof(size));
+    assert(array.size() >= sizeof(size) + size);
+    return {array.data() + sizeof(size), size};
+}
+
+EntryRef
+RawBufferStore::set(vespalib::ConstArrayRef<char> raw)
+{
+    uint32_t size = raw.size();
+    if (size == 0) {
+        return EntryRef();
+    }
+    size_t buffer_size = raw.size() + sizeof(size);
+    auto& mapper = _array_store.get_mapper();
+    auto type_id = mapper.get_type_id(buffer_size);
+    auto array_size = (type_id != 0) ? mapper.get_array_size(type_id) : buffer_size;
+    assert(array_size >= buffer_size);
+    auto ref = _array_store.allocate(array_size);
+    auto buf = _array_store.get_writable(ref);
+    memcpy(buf.data(), &size, sizeof(size));
+    memcpy(buf.data() + sizeof(size), raw.data(), size);
+    if (array_size > buffer_size) {
+        memset(buf.data() + buffer_size, 0, array_size - buffer_size);
+    }
+    return ref;
+}
+
+std::unique_ptr<vespalib::datastore::ICompactionContext>
+RawBufferStore::start_compact(const vespalib::datastore::CompactionStrategy& compaction_strategy)
+{
+    return _array_store.compact_worst(compaction_strategy);
+}
+
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.h b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.h
new file mode 100644
index 00000000000..60132c70852
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.h
@@ -0,0 +1,35 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/datastore/array_store.h>
+#include "raw_buffer_type_mapper.h"
+
+namespace search::attribute {
+
+/**
+ * Class handling storage of raw values in an array store. A stored entry
+ * starts with 4 bytes that contains the size of the raw value.
+ */
+class RawBufferStore
+{
+    using EntryRef = vespalib::datastore::EntryRef;
+    using RefType = vespalib::datastore::EntryRefT<19>;
+    using ArrayStoreType = vespalib::datastore::ArrayStore<char, RefType, RawBufferTypeMapper>;
+    using generation_t = vespalib::GenerationHandler::generation_t;
+
+    ArrayStoreType                      _array_store;
+public:
+    RawBufferStore(std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator, uint32_t max_small_buffer_type_id, double grow_factor);
+    ~RawBufferStore();
+    EntryRef set(vespalib::ConstArrayRef<char> raw);
+    vespalib::ConstArrayRef<char> get(EntryRef ref) const;
+    void remove(EntryRef ref) { _array_store.remove(ref); }
+    vespalib::MemoryUsage update_stat(const vespalib::datastore::CompactionStrategy& compaction_strategy) { return _array_store.update_stat(compaction_strategy); }
+    bool consider_compact() const noexcept { return _array_store.consider_compact(); }
+    std::unique_ptr<vespalib::datastore::ICompactionContext> start_compact(const vespalib::datastore::CompactionStrategy& compaction_strategy);
+    void reclaim_memory(generation_t oldest_used_gen) { _array_store.reclaim_memory(oldest_used_gen); }
+    void assign_generation(generation_t current_gen) { _array_store.assign_generation(current_gen); }
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp
index a37400ed88e..9746929c666 100644
--- a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp
@@ -9,8 +9,6 @@ using vespalib::datastore::EntryRef;
 
 namespace {
 
-constexpr float ALLOC_GROW_FACTOR = 0.2;
-
 constexpr double mapper_grow_factor = 1.03;
 
 constexpr uint32_t max_small_buffer_type_id = 500u;
@@ -22,12 +20,7 @@ namespace search::attribute {
 SingleRawAttribute::SingleRawAttribute(const vespalib::string& name, const Config& config)
     : NotImplementedAttribute(name, config),
       _ref_vector(config.getGrowStrategy(), getGenerationHolder()),
-      _array_store(ArrayStoreType::optimizedConfigForHugePage(max_small_buffer_type_id,
-                                                              RawBufferTypeMapper(max_small_buffer_type_id, mapper_grow_factor),
-                                                              MemoryAllocator::HUGEPAGE_SIZE,
-                                                              MemoryAllocator::PAGE_SIZE,
-                                                              8_Ki, ALLOC_GROW_FACTOR),
-                   get_memory_allocator(), RawBufferTypeMapper(max_small_buffer_type_id, mapper_grow_factor))
+      _raw_store(get_memory_allocator(), max_small_buffer_type_id, mapper_grow_factor)
 {
 }
 
@@ -39,7 +32,7 @@ SingleRawAttribute::~SingleRawAttribute()
 void
 SingleRawAttribute::reclaim_memory(generation_t oldest_used_gen)
 {
-    _array_store.reclaim_memory(oldest_used_gen);
+    _raw_store.reclaim_memory(oldest_used_gen);
     getGenerationHolder().reclaim(oldest_used_gen);
 }
 
@@ -47,7 +40,7 @@ void
 SingleRawAttribute::before_inc_generation(generation_t current_gen)
 {
     getGenerationHolder().assign_generation(current_gen);
-    _array_store.assign_generation(current_gen);
+    _raw_store.assign_generation(current_gen);
 }
 
 bool
@@ -70,8 +63,8 @@ void
 SingleRawAttribute::onCommit()
 {
     incGeneration();
-    if (_array_store.consider_compact()) {
-        auto context = _array_store.compact_worst(getConfig().getCompactionStrategy());
+    if (_raw_store.consider_compact()) {
+        auto context = _raw_store.start_compact(getConfig().getCompactionStrategy());
         if (context) {
             context->compact(vespalib::ArrayRef<AtomicEntryRef>(&_ref_vector[0], _ref_vector.size()));
         }
@@ -96,23 +89,12 @@ vespalib::MemoryUsage
 SingleRawAttribute::update_stat()
 {
     vespalib::MemoryUsage result = _ref_vector.getMemoryUsage();
-    result.merge(_array_store.update_stat(getConfig().getCompactionStrategy()));
+    result.merge(_raw_store.update_stat(getConfig().getCompactionStrategy()));
     result.mergeGenerationHeldBytes(getGenerationHolder().get_held_bytes());
     return result;
 }
 
 vespalib::ConstArrayRef<char>
-SingleRawAttribute::get_raw(EntryRef ref) const
-{
-    auto array = _array_store.get(ref);
-    uint32_t size = 0;
-    assert(array.size() >= sizeof(size));
-    memcpy(&size, array.data(), sizeof(size));
-    assert(array.size() >= sizeof(size) + size);
-    return {array.data() + sizeof(size), size};
-}
-
-vespalib::ConstArrayRef<char>
 SingleRawAttribute::get_raw(DocId docid) const
 {
     EntryRef ref;
@@ -122,42 +104,20 @@ SingleRawAttribute::get_raw(DocId docid) const
     if (!ref.valid()) {
         return {};
     }
-    return get_raw(ref);
-}
-
-EntryRef
-SingleRawAttribute::set_raw(vespalib::ConstArrayRef<char> raw)
-{
-    uint32_t size = raw.size();
-    if (size == 0) {
-        return EntryRef();
-    }
-    size_t buffer_size = raw.size() + sizeof(size);
-    auto& mapper = _array_store.get_mapper();
-    auto type_id = mapper.get_type_id(buffer_size);
-    auto array_size = (type_id != 0) ? mapper.get_array_size(type_id) : buffer_size;
-    assert(array_size >= buffer_size);
-    auto ref = _array_store.allocate(array_size);
-    auto buf = _array_store.get_writable(ref);
-    memcpy(buf.data(), &size, sizeof(size));
-    memcpy(buf.data() + sizeof(size), raw.data(), size);
-    if (array_size > buffer_size) {
-        memset(buf.data() + buffer_size, 0, array_size - buffer_size);
-    }
-    return ref;
+    return _raw_store.get(ref);
 }
 
 void
 SingleRawAttribute::set_raw(DocId docid, vespalib::ConstArrayRef<char> raw)
 {
-    auto ref = set_raw(raw);
+    auto ref = _raw_store.set(raw);
     assert(docid < _ref_vector.size());
     updateUncommittedDocIdLimit(docid);
     auto& elem_ref = _ref_vector[docid];
     EntryRef old_ref(elem_ref.load_relaxed());
     elem_ref.store_release(ref);
     if (old_ref.valid()) {
-        _array_store.remove(old_ref);
+        _raw_store.remove(old_ref);
     }
 }
 
@@ -169,7 +129,7 @@ SingleRawAttribute::clearDoc(DocId docId)
     EntryRef old_ref(elem_ref.load_relaxed());
     elem_ref.store_relaxed(EntryRef());
     if (old_ref.valid()) {
-        _array_store.remove(old_ref);
+        _raw_store.remove(old_ref);
         return 1u;
     }
     return 0u;
diff --git a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h
index 876acc9ad58..d7ea321a3d4 100644
--- a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h
+++ b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.h
@@ -3,8 +3,7 @@
 #pragma once
 
 #include "not_implemented_attribute.h"
-#include "raw_buffer_type_mapper.h"
-#include <vespa/vespalib/datastore/array_store.h>
+#include "raw_buffer_store.h"
 #include <vespa/vespalib/util/rcuvector.h>
 
 namespace search::attribute {
@@ -17,17 +16,12 @@ class SingleRawAttribute : public NotImplementedAttribute
     using AtomicEntryRef = vespalib::datastore::AtomicEntryRef;
     using EntryRef = vespalib::datastore::EntryRef;
     using RefVector = vespalib::RcuVectorBase<AtomicEntryRef>;
-    using RefType = vespalib::datastore::EntryRefT<19>;
-    using ArrayStoreType = vespalib::datastore::ArrayStore<char, RefType, RawBufferTypeMapper>;
 
     RefVector                           _ref_vector;
-    ArrayStoreType                      _array_store;
-    vespalib::datastore::CompactionSpec _compaction_spec;
+    RawBufferStore                      _raw_store;
 
     vespalib::MemoryUsage update_stat();
     EntryRef acquire_entry_ref(DocId docid) const noexcept { return _ref_vector.acquire_elem_ref(docid).load_acquire(); }
-    EntryRef set_raw(vespalib::ConstArrayRef<char> raw);
-    vespalib::ConstArrayRef<char> get_raw(EntryRef ref) const;
 public:
     SingleRawAttribute(const vespalib::string& name, const Config& config);
     ~SingleRawAttribute() override;