diff options
Diffstat (limited to 'searchlib/src/vespa')
7 files changed, 48 insertions, 14 deletions
diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp index c4dd0ef30f0..6612db1d27e 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp @@ -15,7 +15,7 @@ namespace search::tensor { SerializedFastValueAttribute::SerializedFastValueAttribute(stringref name, const Config &cfg, const NearestNeighborIndexFactory& index_factory) : TensorAttribute(name, cfg, _tensorBufferStore, index_factory), - _tensorBufferStore(cfg.tensorType(), get_memory_allocator(), 1000u) + _tensorBufferStore(cfg.tensorType(), get_memory_allocator(), 400u) { } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp index 3a42b47ace3..fcdb9311ec6 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp @@ -110,7 +110,7 @@ TensorBufferOperations::store_tensor(ArrayRef<char> buf, const vespalib::eval::V auto cells_start_offset = aligner.align(labels_end_offset); auto cells_end_offset = cells_start_offset + cells_mem_size; auto store_end = aligner.align(cells_end_offset); - assert(store_end == get_array_size(num_subspaces)); + assert(store_end == get_buffer_size(num_subspaces)); assert(buf.size() >= store_end); *reinterpret_cast<uint32_t*>(buf.data()) = num_subspaces; auto labels = reinterpret_cast<string_id*>(buf.data() + get_labels_offset()); @@ -137,8 +137,8 @@ TensorBufferOperations::store_tensor(ArrayRef<char> buf, const vespalib::eval::V if (cells_mem_size > 0) { memcpy(buf.data() + cells_start_offset, cells.data, cells_mem_size); } - if (cells_end_offset != store_end) { - memset(buf.data() + cells_end_offset, 0, store_end - cells_end_offset); + if (cells_end_offset != buf.size()) { + memset(buf.data() + cells_end_offset, 0, buf.size() - cells_end_offset); } } @@ -146,7 +146,7 @@ std::unique_ptr<vespalib::eval::Value> TensorBufferOperations::make_fast_view(ConstArrayRef<char> buf, const vespalib::eval::ValueType& tensor_type) const { auto num_subspaces = get_num_subspaces(buf); - assert(buf.size() >= get_array_size(num_subspaces)); + assert(buf.size() >= get_buffer_size(num_subspaces)); ConstArrayRef<string_id> labels(reinterpret_cast<const string_id*>(buf.data() + get_labels_offset()), num_subspaces * _num_mapped_dimensions); auto cells_size = num_subspaces * _subspace_type.size(); auto cells_mem_size = num_subspaces * _subspace_type.mem_size(); // Size measured in bytes @@ -185,7 +185,7 @@ void TensorBufferOperations::encode_stored_tensor(ConstArrayRef<char> buf, const vespalib::eval::ValueType& tensor_type, vespalib::nbostream& target) const { auto num_subspaces = get_num_subspaces(buf); - assert(buf.size() >= get_array_size(num_subspaces)); + assert(buf.size() >= get_buffer_size(num_subspaces)); ConstArrayRef<string_id> labels(reinterpret_cast<const string_id*>(buf.data() + get_labels_offset()), num_subspaces * _num_mapped_dimensions); auto cells_size = num_subspaces * _subspace_type.size(); auto cells_mem_size = num_subspaces * _subspace_type.mem_size(); // Size measured in bytes diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h index 26cf9a429a1..3928b41c2d1 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h @@ -80,7 +80,8 @@ class TensorBufferOperations return get_num_subspaces(get_num_subspaces_and_flag(buf)); } public: - size_t get_array_size(uint32_t num_subspaces) const noexcept { + // Size (in bytes) used to serialize tensor with num_subspaces. + size_t get_buffer_size(uint32_t num_subspaces) const noexcept { auto cells_mem_size = get_cells_mem_size(num_subspaces); auto aligner = select_aligner(cells_mem_size); return get_cells_offset(num_subspaces, aligner) + aligner.align(cells_mem_size); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp index 6c2ef698dc0..ff39c33fc5d 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp @@ -26,6 +26,8 @@ namespace { constexpr float ALLOC_GROW_FACTOR = 0.2; +constexpr double mapper_grow_factor = 1.02; + } TensorBufferStore::TensorBufferStore(const ValueType& tensor_type, std::shared_ptr<MemoryAllocator> allocator, uint32_t max_small_subspaces_type_id) @@ -33,11 +35,11 @@ TensorBufferStore::TensorBufferStore(const ValueType& tensor_type, std::shared_p _tensor_type(tensor_type), _ops(_tensor_type), _array_store(ArrayStoreType::optimizedConfigForHugePage(max_small_subspaces_type_id, - TensorBufferTypeMapper(max_small_subspaces_type_id, &_ops), + TensorBufferTypeMapper(max_small_subspaces_type_id, mapper_grow_factor, &_ops), MemoryAllocator::HUGEPAGE_SIZE, MemoryAllocator::PAGE_SIZE, 8_Ki, ALLOC_GROW_FACTOR), - std::move(allocator), TensorBufferTypeMapper(max_small_subspaces_type_id, &_ops)) + std::move(allocator), TensorBufferTypeMapper(max_small_subspaces_type_id, mapper_grow_factor, &_ops)) { } @@ -81,7 +83,11 @@ EntryRef TensorBufferStore::store_tensor(const Value &tensor) { uint32_t num_subspaces = tensor.index().size(); - auto array_size = _ops.get_array_size(num_subspaces); + auto buffer_size = _ops.get_buffer_size(num_subspaces); + auto& mapper = _array_store.get_mapper(); + auto type_id = mapper.get_type_id(buffer_size); + auto array_size = (type_id != 0) ? mapper.get_array_size(type_id) : buffer_size; + assert(array_size >= buffer_size); auto ref = _array_store.allocate(array_size); auto buf = _array_store.get_writable(ref); _ops.store_tensor(buf, tensor); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h index ce00977c298..f602836bd32 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h @@ -44,6 +44,9 @@ public: auto buf = _array_store.get(ref); return _ops.get_vectors(buf); } + + // Used by unit test + static constexpr uint32_t get_offset_bits() noexcept { return RefType::offset_bits; } }; } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp index b4b0b9bbc79..ce8cc11026c 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp @@ -3,6 +3,8 @@ #include "tensor_buffer_type_mapper.h" #include "tensor_buffer_operations.h" #include <algorithm> +#include <cmath> +#include <limits> namespace search::tensor { @@ -12,15 +14,29 @@ TensorBufferTypeMapper::TensorBufferTypeMapper() { } -TensorBufferTypeMapper::TensorBufferTypeMapper(uint32_t max_small_subspaces_type_id, TensorBufferOperations* ops) +TensorBufferTypeMapper::TensorBufferTypeMapper(uint32_t max_small_subspaces_type_id, double grow_factor, TensorBufferOperations* ops) : _array_sizes(), _ops(ops) { _array_sizes.reserve(max_small_subspaces_type_id + 1); _array_sizes.emplace_back(0); // type id 0 uses LargeSubspacesBufferType + uint32_t num_subspaces = 0; + size_t prev_array_size = 0u; + size_t array_size = 0u; for (uint32_t type_id = 1; type_id <= max_small_subspaces_type_id; ++type_id) { - auto num_subspaces = type_id - 1; - _array_sizes.emplace_back(_ops->get_array_size(num_subspaces)); + if (type_id > 1) { + num_subspaces = std::max(num_subspaces + 1, static_cast<uint32_t>(std::floor(num_subspaces * grow_factor))); + } + array_size = _ops->get_buffer_size(num_subspaces); + while (array_size <= prev_array_size) { + ++num_subspaces; + array_size = _ops->get_buffer_size(num_subspaces); + } + if (array_size > std::numeric_limits<uint32_t>::max()) { + break; + } + _array_sizes.emplace_back(array_size); + prev_array_size = array_size; } } @@ -44,4 +60,11 @@ TensorBufferTypeMapper::get_array_size(uint32_t type_id) const return _array_sizes[type_id]; } +uint32_t +TensorBufferTypeMapper::get_max_small_array_type_id(uint32_t max_small_array_type_id) const noexcept +{ + auto clamp_type_id = _array_sizes.size() - 1; + return (clamp_type_id < max_small_array_type_id) ? clamp_type_id : max_small_array_type_id; +} + } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h index 950076c9924..ad2116a429c 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h @@ -25,12 +25,13 @@ public: using LargeBufferType = LargeSubspacesBufferType; TensorBufferTypeMapper(); - TensorBufferTypeMapper(uint32_t max_small_subspaces_type_id, TensorBufferOperations* ops); + TensorBufferTypeMapper(uint32_t max_small_subspaces_type_id, double grow_factor, TensorBufferOperations* ops); ~TensorBufferTypeMapper(); uint32_t get_type_id(size_t array_size) const; size_t get_array_size(uint32_t type_id) const; TensorBufferOperations& get_tensor_buffer_operations() const noexcept { return *_ops; } + uint32_t get_max_small_array_type_id(uint32_t max_small_array_type_id) const noexcept; }; } |