diff options
9 files changed, 80 insertions, 36 deletions
diff --git a/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp index 1ca87467fc6..29c032664c0 100644 --- a/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp @@ -15,17 +15,19 @@ TEST(WordStoreTest, words_can_be_added_and_retrieved) std::string w2 = "that"; std::string w3 = "words"; WordStore ws; + static constexpr auto buffer_array_size = WordStore::buffer_array_size; + using Aligner = WordStore::Aligner; EntryRef r1 = ws.addWord(w1); EntryRef r2 = ws.addWord(w2); EntryRef r3 = ws.addWord(w3); uint32_t invp = WordStore::buffer_array_size; // Reserved as invalid uint32_t w1s = w1.size() + 1; - uint32_t w1p = WordStore::calc_pad(w1s); + uint32_t w1p = Aligner::pad(w1s); uint32_t w2s = w2.size() + 1; - uint32_t w2p = WordStore::calc_pad(w2s); - EXPECT_EQ(invp, WordStore::RefType(r1).offset() * WordStore::buffer_array_size); - EXPECT_EQ(invp + w1s + w1p, WordStore::RefType(r2).offset() * WordStore::buffer_array_size); - EXPECT_EQ(invp + w1s + w1p + w2s + w2p, WordStore::RefType(r3).offset() * WordStore::buffer_array_size); + uint32_t w2p = Aligner::pad(w2s); + EXPECT_EQ(invp, WordStore::RefType(r1).offset() * buffer_array_size); + EXPECT_EQ(invp + w1s + w1p, WordStore::RefType(r2).offset() * buffer_array_size); + EXPECT_EQ(invp + w1s + w1p + w2s + w2p, WordStore::RefType(r3).offset() * buffer_array_size); EXPECT_EQ(0u, WordStore::RefType(r1).bufferId()); EXPECT_EQ(0u, WordStore::RefType(r2).bufferId()); EXPECT_EQ(0u, WordStore::RefType(r3).bufferId()); diff --git a/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp index b5810d06047..d2a28ebc04a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp @@ -30,7 +30,7 @@ FeatureStore::writeFeatures(uint32_t packedIndex, const DocIdAndFeatures &featur EntryRef FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen) { - uint32_t pad = calc_pad(byteLen); + uint32_t pad = Aligner::pad(byteLen); auto result = _store.rawAllocator<uint8_t>(_typeId).alloc(byteLen + pad, DECODE_SAFETY); uint8_t *dst = result.data; memcpy(dst, src, byteLen); @@ -64,7 +64,7 @@ FeatureStore::moveFeatures(EntryRef ref, uint64_t bitLen) uint64_t byteLen = (bitLen + 7) / 8; EntryRef newRef = addFeatures(src, byteLen); // Mark old features as dead - _store.incDead(ref, byteLen + calc_pad(byteLen)); + _store.incDead(ref, byteLen + Aligner::pad(byteLen)); return newRef; } @@ -109,7 +109,7 @@ void FeatureStore::add_features_guard_bytes() { uint32_t len = DECODE_SAFETY; - uint32_t pad = calc_pad(len); + uint32_t pad = Aligner::pad(len); auto result = _store.rawAllocator<uint8_t>(_typeId).alloc(len + pad); memset(result.data, 0, len + pad); _store.incDead(result.ref, len + pad); diff --git a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h index b1d975d0926..3ecb61f1cd1 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/vespalib/datastore/aligner.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/bitcompression/posocc_fields_params.h> @@ -20,8 +21,7 @@ public: using DecodeContextCooked = bitcompression::EG2PosOccDecodeContextCooked<true>; using generation_t = vespalib::GenerationHandler::generation_t; static constexpr uint32_t buffer_array_size = 4u; // Must be a power of 2 - static constexpr uint32_t pad_constant = buffer_array_size - 1u; - static uint32_t calc_pad(uint32_t val) { return (-val & pad_constant); } + using Aligner = vespalib::datastore::Aligner<buffer_array_size>; private: using Schema = index::Schema; diff --git a/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp index 441587eb718..3e4c38ceb0e 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp @@ -26,7 +26,7 @@ vespalib::datastore::EntryRef WordStore::addWord(const vespalib::stringref word) { size_t wordSize = word.size() + 1; - size_t bufferSize = wordSize + calc_pad(wordSize); + size_t bufferSize = wordSize + Aligner::pad(wordSize); auto result = _store.rawAllocator<char>(_typeId).alloc(bufferSize); char *be = result.data; for (size_t i = 0; i < word.size(); ++i) { diff --git a/searchlib/src/vespa/searchlib/memoryindex/word_store.h b/searchlib/src/vespa/searchlib/memoryindex/word_store.h index 913f6bc3ea5..896bbf5d75e 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/word_store.h +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/vespalib/datastore/aligner.h> #include <vespa/vespalib/datastore/datastore.h> #include <vespa/vespalib/stllike/string.h> @@ -12,8 +13,7 @@ public: using DataStoreType = vespalib::datastore::DataStoreT<vespalib::datastore::EntryRefT<22>>; using RefType = DataStoreType::RefType; static constexpr uint32_t buffer_array_size = 4u; // Must be a power of 2 - static constexpr uint32_t pad_constant = buffer_array_size - 1u; - static uint32_t calc_pad(uint32_t val) { return (-val & pad_constant); } + using Aligner = vespalib::datastore::Aligner<buffer_array_size>; private: DataStoreType _store; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp index 807975c392a..22298354444 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp @@ -98,10 +98,10 @@ TensorBufferOperations::store_tensor(ArrayRef<char> buf, const vespalib::eval::V auto labels_end_offset = get_labels_offset() + get_labels_mem_size(num_subspaces); auto cells_size = num_subspaces * _dense_subspace_size; auto cells_mem_size = cells_size * _cell_mem_size; // Size measured in bytes - auto alignment = select_alignment(cells_mem_size); - auto cells_start_offset = calc_aligned(labels_end_offset, alignment); + auto aligner = select_aligner(cells_mem_size); + auto cells_start_offset = aligner.align(labels_end_offset); auto cells_end_offset = cells_start_offset + cells_mem_size; - auto store_end = calc_aligned(cells_end_offset, alignment); + auto store_end = aligner.align(cells_end_offset); assert(store_end == get_array_size(num_subspaces)); assert(buf.size() >= store_end); *reinterpret_cast<uint32_t*>(buf.data()) = num_subspaces; @@ -140,8 +140,8 @@ TensorBufferOperations::make_fast_view(ConstArrayRef<char> buf, const vespalib:: ConstArrayRef<string_id> labels(reinterpret_cast<const string_id*>(buf.data() + get_labels_offset()), num_subspaces * _num_mapped_dimensions); auto cells_size = num_subspaces * _dense_subspace_size; auto cells_mem_size = cells_size * _cell_mem_size; // Size measured in bytes - auto alignment = select_alignment(cells_mem_size); - auto cells_start_offset = get_cells_offset(num_subspaces, alignment); + auto aligner = select_aligner(cells_mem_size); + auto cells_start_offset = get_cells_offset(num_subspaces, aligner); TypedCells cells(buf.data() + cells_start_offset, _cell_type, cells_size); assert(cells_start_offset + cells_mem_size <= buf.size()); return std::make_unique<FastValueView>(tensor_type, labels, cells, _num_mapped_dimensions, num_subspaces); @@ -176,8 +176,8 @@ TensorBufferOperations::encode_stored_tensor(ConstArrayRef<char> buf, const vesp ConstArrayRef<string_id> labels(reinterpret_cast<const string_id*>(buf.data() + get_labels_offset()), num_subspaces * _num_mapped_dimensions); auto cells_size = num_subspaces * _dense_subspace_size; auto cells_mem_size = cells_size * _cell_mem_size; // Size measured in bytes - auto alignment = select_alignment(cells_mem_size); - auto cells_start_offset = get_cells_offset(num_subspaces, alignment); + auto aligner = select_aligner(cells_mem_size); + auto cells_start_offset = get_cells_offset(num_subspaces, aligner); TypedCells cells(buf.data() + cells_start_offset, _cell_type, cells_size); assert(cells_start_offset + cells_mem_size <= buf.size()); StringIdVector labels_copy(labels.begin(), labels.end()); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h index ab3777d7526..18c7bc84ab2 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/eval/eval/cell_type.h> +#include <vespa/vespalib/datastore/aligner.h> #include <vespa/vespalib/util/string_id.h> #include <cstddef> #include <memory> @@ -45,32 +46,31 @@ class TensorBufferOperations std::vector<vespalib::string_id> _addr; std::vector<vespalib::string_id*> _addr_refs; + using Aligner = vespalib::datastore::Aligner<vespalib::datastore::dynamic_alignment>; + static constexpr size_t CELLS_ALIGNMENT = 16; static constexpr size_t CELLS_ALIGNMENT_MEM_SIZE_MIN = 32; static constexpr size_t get_num_subspaces_size() noexcept { return sizeof(uint32_t); } static constexpr size_t get_labels_offset() noexcept { return get_num_subspaces_size(); } - static size_t calc_aligned(size_t unaligned, size_t alignment) noexcept { - return (unaligned + alignment - 1) & (- alignment); - } size_t get_cells_mem_size(uint32_t num_subspaces) const noexcept { return _dense_subspace_size * _cell_mem_size * num_subspaces; } - size_t select_alignment(size_t cells_mem_size) const noexcept { - return (cells_mem_size < CELLS_ALIGNMENT_MEM_SIZE_MIN) ? _min_alignment : CELLS_ALIGNMENT; + auto select_aligner(size_t cells_mem_size) const noexcept { + return Aligner((cells_mem_size < CELLS_ALIGNMENT_MEM_SIZE_MIN) ? _min_alignment : CELLS_ALIGNMENT); } size_t get_labels_mem_size(uint32_t num_subspaces) const noexcept { return sizeof(vespalib::string_id) * _num_mapped_dimensions * num_subspaces; } - size_t get_cells_offset(uint32_t num_subspaces, size_t alignment) const noexcept { - return calc_aligned(get_labels_offset() + get_labels_mem_size(num_subspaces), alignment); + size_t get_cells_offset(uint32_t num_subspaces, auto aligner) const noexcept { + return aligner.align(get_labels_offset() + get_labels_mem_size(num_subspaces)); } uint32_t get_num_subspaces(vespalib::ConstArrayRef<char> buf) const noexcept; public: size_t get_array_size(uint32_t num_subspaces) const noexcept { auto cells_mem_size = get_cells_mem_size(num_subspaces); - auto alignment = select_alignment(cells_mem_size); - return get_cells_offset(num_subspaces, alignment) + calc_aligned(cells_mem_size, alignment); + auto aligner = select_aligner(cells_mem_size); + return get_cells_offset(num_subspaces, aligner) + aligner.align(cells_mem_size); } TensorBufferOperations(const vespalib::eval::ValueType& tensor_type); ~TensorBufferOperations(); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp index 11b6a1e3020..2597aec3dd7 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp @@ -199,14 +199,12 @@ FakeMemTreeOccMgr::sync() void FakeMemTreeOccMgr::add(uint32_t wordIdx, index::DocIdAndFeatures &features) { + using Aligner = FeatureStore::Aligner; const FakeWord *fw = _fakeWords[wordIdx]; std::pair<EntryRef, uint64_t> r = _featureStore.addFeatures(fw->getPackedIndex(), features); - size_t feature_size = (r.second + 7) / 8; - feature_size += FeatureStore::calc_pad(feature_size); - - _featureSizes[wordIdx] += feature_size * 8; + _featureSizes[wordIdx] += Aligner::align((r.second + 7) / 8) * 8; _unflushed.push_back(PendingOp(wordIdx, features.doc_id(), r.first)); @@ -240,6 +238,7 @@ FakeMemTreeOccMgr::sortUnflushed() void FakeMemTreeOccMgr::flush() { + using Aligner = FeatureStore::Aligner; typedef std::vector<PendingOp>::iterator I; if (_unflushed.empty()) @@ -263,9 +262,7 @@ FakeMemTreeOccMgr::flush() if (i->getRemove()) { if (itr.valid() && itr.getKey() == docId) { uint64_t bits = _featureStore.bitSize(fw->getPackedIndex(), EntryRef(itr.getData().get_features_relaxed())); - size_t feature_size = (bits + 7) / 8; - feature_size += FeatureStore::calc_pad(feature_size); - _featureSizes[wordIdx] -= feature_size * 8; + _featureSizes[wordIdx] -= Aligner::align((bits + 7) / 8) * 8; tree.remove(itr); } } else { diff --git a/vespalib/src/vespa/vespalib/datastore/aligner.h b/vespalib/src/vespa/vespalib/datastore/aligner.h new file mode 100644 index 00000000000..c9879046dd6 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/aligner.h @@ -0,0 +1,45 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstddef> +#include <limits> + +namespace vespalib::datastore { + +inline constexpr size_t dynamic_alignment = std::numeric_limits<size_t>::max(); + +/* + * Class used to align offsets to specified alignment. + * + * Alignment template parameter must be a power of 2 or the + * dynamic_alignment value (which triggers specialization below). + */ +template <size_t alignment_v = dynamic_alignment> +class Aligner { +public: + explicit constexpr Aligner() = default; + explicit constexpr Aligner(size_t); // Never used but must be declared + static size_t align(size_t unaligned) noexcept { return (unaligned + alignment_v - 1) & (- alignment_v); } + static size_t pad(size_t unaligned) noexcept { return (- unaligned & (alignment_v - 1)); } + static size_t alignment() noexcept { return alignment_v; } +}; + +/* + * Specialization when alignment template argument is dynamic_alignment. + * The constructor argument must be a power of 2. + */ +template <> +class Aligner<dynamic_alignment> { + size_t _alignment; +public: + explicit constexpr Aligner(size_t alignment_) + : _alignment(alignment_) + { + } + size_t align(size_t unaligned) const noexcept { return (unaligned + _alignment - 1) & (- _alignment); } + size_t pad(size_t unaligned) const noexcept { return (- unaligned & (_alignment - 1)); } + size_t alignment() const noexcept { return _alignment; } +}; + +} |