diff options
44 files changed, 419 insertions, 213 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 2b36067a65c..2225671d8f8 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -4,6 +4,7 @@ #include "documentmetastoresaver.h" #include "operation_listener.h" #include "search_context.h" +#include "document_meta_store_versions.h" #include <vespa/fastos/file.h> #include <vespa/persistence/spi/bucket_limits.h> #include <vespa/searchcore/proton/bucketdb/bucketsessionbase.h> @@ -21,7 +22,7 @@ #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/rcuvector.hpp> -#include "document_meta_store_versions.h" +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/log/log.h> LOG_SETUP(".proton.documentmetastore"); @@ -70,15 +71,15 @@ private: public: Reader(std::unique_ptr<FastOS_FileInterface> datFile) - : _datFile(std::move(datFile)), - _lidReader(*_datFile), - _gidReader(*_datFile), - _bucketUsedBitsReader(*_datFile), - _timestampReader(*_datFile), - _header(), - _headerLen(0u), - _docIdLimit(0), - _datFileSize(0u) + : _datFile(std::move(datFile)), + _lidReader(*_datFile), + _gidReader(*_datFile), + _bucketUsedBitsReader(*_datFile), + _timestampReader(*_datFile), + _header(), + _headerLen(0u), + _docIdLimit(0), + _datFileSize(0u) { _headerLen = _header.readFile(*_datFile); _datFile->SetPosition(_headerLen); diff --git a/searchlib/src/tests/btree/scanspeed.cpp b/searchlib/src/tests/btree/scanspeed.cpp index 1474edd6b0b..ed3368f4b4e 100644 --- a/searchlib/src/tests/btree/scanspeed.cpp +++ b/searchlib/src/tests/btree/scanspeed.cpp @@ -13,6 +13,7 @@ #include <vespa/vespalib/btree/btreebuilder.hpp> #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreestore.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/util/time.h> #include <vespa/searchlib/common/bitvector.h> diff --git a/searchlib/src/tests/predicate/simple_index_test.cpp b/searchlib/src/tests/predicate/simple_index_test.cpp index c5177c526c0..f7398800dba 100644 --- a/searchlib/src/tests/predicate/simple_index_test.cpp +++ b/searchlib/src/tests/predicate/simple_index_test.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/btree/btreenodeallocator.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <map> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index 19d30317c7b..304a16c492b 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -310,3 +310,11 @@ class BTree<IEnumStore::Index, uint32_t, NoAggregated, const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; } + +namespace vespalib::datastore { + +using namespace btree; + +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_INTERNALNODE(EntryRef, NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS); + +}
\ No newline at end of file diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp index 74e277b2859..0ad8a7d7c5b 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp @@ -89,4 +89,3 @@ template class EnumStoreT<double>; namespace vespalib { template class RcuVectorBase<search::IEnumStore::Index>; } - diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp index 6a1a5194890..71f2bcf4d91 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp @@ -6,12 +6,12 @@ #include "multi_value_mapping.hpp" #include "multivalue.h" #include <vespa/vespalib/util/array.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> using search::multivalue::Value; using search::multivalue::WeightedValue; -namespace search { -namespace attribute { +namespace search::attribute { template class MultiValueMapping<Value<IEnumStore::Index>>; template class MultiValueMapping<WeightedValue<IEnumStore::Index>>; @@ -28,5 +28,4 @@ template class MultiValueMapping<WeightedValue<float>>; template class MultiValueMapping<Value<double>>; template class MultiValueMapping<WeightedValue<double>>; -} // namespace search::attribute -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/postinglisttraits.cpp b/searchlib/src/vespa/searchlib/attribute/postinglisttraits.cpp index ae872ee60f1..1dd6631080a 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglisttraits.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglisttraits.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/btree/btreebuilder.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> namespace vespalib::btree { diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 26339392d2c..d06b891c2e2 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -6,6 +6,7 @@ #include <vespa/searchcommon/attribute/status.h> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/datastore/datastore.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> namespace search::attribute { @@ -255,7 +256,7 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref) uint32_t expDocFreq = it.size(); (void) expDocFreq; for (; it.valid(); ++it) { - uint32_t docId = it.getKey(); + uint32_t docId = it.getKey(); assert(docId < docIdLimit); bv.setBit(docId); } @@ -278,7 +279,7 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref) ref = bPair.ref; } - + template <typename DataT> void PostingStore<DataT>::applyNewBitVector(EntryRef &ref, @@ -314,7 +315,7 @@ PostingStore<DataT>::applyNewBitVector(EntryRef &ref, ref = bPair.ref; } - + template <typename DataT> void PostingStore<DataT>::apply(BitVector &bv, @@ -628,3 +629,10 @@ template class PostingStore<BTreeNoLeafData>; template class PostingStore<int32_t>; } + +namespace vespalib::datastore { + using namespace btree; + + + +} diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index 51e15c26b8f..fd4b0365ca1 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -11,9 +11,10 @@ #include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h> #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/datastore/unique_store_builder.h> #include <vespa/vespalib/datastore/datastore.hpp> #include <vespa/vespalib/datastore/unique_store.hpp> -#include <vespa/vespalib/datastore/unique_store_builder.h> +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.reference_attribute"); diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index fecb8116f90..c052c9410e2 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -15,6 +15,7 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/util/array.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchlib/src/vespa/searchlib/predicate/simple_index.cpp b/searchlib/src/vespa/searchlib/predicate/simple_index.cpp index 364324aeb97..1b0db8f52d4 100644 --- a/searchlib/src/vespa/searchlib/predicate/simple_index.cpp +++ b/searchlib/src/vespa/searchlib/predicate/simple_index.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/util/array.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/log/log.h> LOG_SETUP(".searchlib.predicate.simple_index"); diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index bc8362c2643..bbffc7ec4a5 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -54,7 +54,7 @@ DenseTensorStore::BufferType::~BufferType() = default; void DenseTensorStore::BufferType::cleanHold(void *buffer, size_t offset, - size_t numElems, CleanContext) + ElemCount numElems, CleanContext) { memset(static_cast<char *>(buffer) + offset, 0, numElems); } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index 49e8a585fec..3f27b4c5218 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -37,7 +37,7 @@ public: public: BufferType(const TensorSizeCalc &tensorSizeCalc); ~BufferType() override; - void cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext cleanCtx) override; + void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; }; private: DataStoreType _concreteStore; diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp index a74da056d00..db0bb7f875f 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp @@ -3,6 +3,7 @@ #include "direct_tensor_store.h" #include <vespa/eval/eval/value.h> #include <vespa/vespalib/datastore/datastore.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> using vespalib::datastore::EntryRef; @@ -16,7 +17,7 @@ DirectTensorStore::TensorBufferType::TensorBufferType() } void -DirectTensorStore::TensorBufferType::cleanHold(void* buffer, size_t offset, size_t num_elems, CleanContext clean_ctx) +DirectTensorStore::TensorBufferType::cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx) { TensorSP* elem = static_cast<TensorSP*>(buffer) + offset; for (size_t i = 0; i < num_elems; ++i) { diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h index f5f2f59d85f..f2867eaa643 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h @@ -28,7 +28,7 @@ private: using CleanContext = typename ParentType::CleanContext; public: TensorBufferType(); - virtual void cleanHold(void* buffer, size_t offset, size_t num_elems, CleanContext clean_ctx) override; + virtual void cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx) override; }; TensorStoreType _tensor_store; diff --git a/storage/src/tests/bucketdb/lockablemaptest.cpp b/storage/src/tests/bucketdb/lockablemaptest.cpp index 7f5069f96b2..fb5c84f217a 100644 --- a/storage/src/tests/bucketdb/lockablemaptest.cpp +++ b/storage/src/tests/bucketdb/lockablemaptest.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/util/document_runnable.h> #include <vespa/storage/bucketdb/btree_lockable_map.hpp> #include <vespa/storage/bucketdb/striped_btree_lockable_map.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/gtest/gtest.h> #include <gmock/gmock.h> diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp index be02db70d9e..c8d8acd629a 100644 --- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp +++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp @@ -3,6 +3,7 @@ #include "btree_bucket_database.h" #include "generic_btree_bucket_database.hpp" #include <vespa/vespalib/datastore/array_store.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <iostream> /* diff --git a/storage/src/vespa/storage/bucketdb/btree_lockable_map.cpp b/storage/src/vespa/storage/bucketdb/btree_lockable_map.cpp index a76f50d41ab..3e5faa90724 100644 --- a/storage/src/vespa/storage/bucketdb/btree_lockable_map.cpp +++ b/storage/src/vespa/storage/bucketdb/btree_lockable_map.cpp @@ -1,8 +1,15 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "btree_lockable_map.hpp" +#include <vespa/vespalib/datastore/buffer_type.hpp> namespace storage::bucketdb { template class BTreeLockableMap<StorageBucketInfo>; // Forced instantiation. } + +namespace vespalib::datastore { + +template class BufferType<storage::bucketdb::StorageBucketInfo>; + +}
\ No newline at end of file diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index 63afd8b770f..5b4c25cb870 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("btree_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <string> #include <vespa/vespalib/btree/btreeroot.h> @@ -18,8 +17,12 @@ LOG_SETUP("btree_test"); #include <vespa/vespalib/btree/btreebuilder.hpp> #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreestore.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/test/btree/btree_printer.h> +#include <vespa/log/log.h> +LOG_SETUP("btree_test"); + using vespalib::GenerationHandler; using vespalib::datastore::EntryRef; diff --git a/vespalib/src/tests/btree/btreeaggregation_test.cpp b/vespalib/src/tests/btree/btreeaggregation_test.cpp index da7f677bb8a..67f5eda4b59 100644 --- a/vespalib/src/tests/btree/btreeaggregation_test.cpp +++ b/vespalib/src/tests/btree/btreeaggregation_test.cpp @@ -14,6 +14,7 @@ #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/btree/btreeaggregator.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/test/btree/btree_printer.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/rand48.h> diff --git a/vespalib/src/tests/btree/frozenbtree_test.cpp b/vespalib/src/tests/btree/frozenbtree_test.cpp index e78c1b40dc6..a56a2d1bcac 100644 --- a/vespalib/src/tests/btree/frozenbtree_test.cpp +++ b/vespalib/src/tests/btree/frozenbtree_test.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreenodeallocator.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/util/rand48.h> #include <map> diff --git a/vespalib/src/tests/btree/iteratespeed.cpp b/vespalib/src/tests/btree/iteratespeed.cpp index e0664b1c588..218e4c9250e 100644 --- a/vespalib/src/tests/btree/iteratespeed.cpp +++ b/vespalib/src/tests/btree/iteratespeed.cpp @@ -13,6 +13,7 @@ #include <vespa/vespalib/btree/btreebuilder.hpp> #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreestore.hpp> +#include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/util/time.h> diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index ea7c746e58f..8ed4ab3767b 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -145,6 +145,15 @@ TEST("require that we test with trivial and non-trivial types") EXPECT_FALSE(vespalib::can_skip_destruction<StringFixture::value_type>::value); } +TEST_F("control static sizes", NumberFixture(3)) { + EXPECT_EQUAL(424u, sizeof(f.store)); + EXPECT_EQUAL(328u, sizeof(NumberFixture::ArrayStoreType::DataStoreType)); + EXPECT_EQUAL(64u, sizeof(NumberFixture::ArrayStoreType::SmallArrayType)); + MemoryUsage usage = f.store.getMemoryUsage(); + EXPECT_EQUAL(960u, usage.allocatedBytes()); + EXPECT_EQUAL(32u, usage.usedBytes()); +} + TEST_F("require that we can add and get small arrays of trivial type", NumberFixture(3)) { TEST_DO(f.assertAdd({})); diff --git a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp index abcf6d3b4e5..96d94e9a71b 100644 --- a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp +++ b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp @@ -11,12 +11,12 @@ constexpr uint32_t MAX_ARRAYS(128); constexpr uint32_t NUM_ARRAYS_FOR_NEW_BUFFER(0); struct Setup { - uint32_t _minArrays; - size_t _usedElems; - size_t _neededElems; - uint32_t _bufferId; - float _allocGrowFactor; - bool _resizing; + uint32_t _minArrays; + ElemCount _usedElems; + ElemCount _neededElems; + uint32_t _bufferId; + float _allocGrowFactor; + bool _resizing; Setup() : _minArrays(0), _usedElems(0), @@ -33,9 +33,9 @@ struct Setup { }; struct Fixture { - Setup setup; + Setup setup; IntBufferType bufferType; - size_t deadElems; + ElemCount deadElems; int buffer[ARRAYS_SIZE]; Fixture(const Setup &setup_) : setup(setup_), diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index a319a423201..406a477949b 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -591,8 +591,9 @@ TEST(DataStoreTest, require_that_offset_in_EntryRefT_is_within_bounds_when_alloc } TEST(DataStoreTest, control_static_sizes) { + EXPECT_EQ(64, sizeof(BufferTypeBase)); EXPECT_EQ(32, sizeof(BufferState::FreeList)); - EXPECT_EQ(4, sizeof(BufferState::State)); + EXPECT_EQ(1, sizeof(BufferState::State)); EXPECT_EQ(128, sizeof(BufferState)); BufferState bs; EXPECT_EQ(0, bs.size()); diff --git a/vespalib/src/vespa/vespalib/btree/btreenode.h b/vespalib/src/vespa/vespalib/btree/btreenode.h index 0c70e70bc6a..b9711cdd05d 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenode.h +++ b/vespalib/src/vespa/vespalib/btree/btreenode.h @@ -14,7 +14,7 @@ namespace vespalib::datastore { template <typename, typename> class Allocator; -template <typename> class BufferType; +template <typename, typename> class BufferType; namespace allocator { template <typename, typename ...> struct Assigner; @@ -63,7 +63,7 @@ protected: ~BTreeNode() { assert(_isFrozen); } public: - typedef datastore::EntryRef Ref; + using Ref = datastore::EntryRef; bool isLeaf() const { return _level == 0u; } bool getFrozen() const { return _isFrozen; } @@ -172,7 +172,7 @@ protected: _keys() {} - ~BTreeNodeT() {} + ~BTreeNodeT() = default; BTreeNodeT(const BTreeNodeT &rhs) : BTreeNode(rhs) @@ -276,8 +276,7 @@ public: }; template <typename KeyT, typename AggrT, uint32_t NumSlots = 16> -class BTreeInternalNode : public BTreeNodeTT<KeyT, BTreeNode::Ref, AggrT, - NumSlots> +class BTreeInternalNode : public BTreeNodeTT<KeyT, BTreeNode::Ref, AggrT, NumSlots> { public: typedef BTreeNodeTT<KeyT, BTreeNode::Ref, AggrT, NumSlots> ParentType; @@ -308,7 +307,7 @@ public: typedef Ref DataType; private: uint32_t _validLeaves; - +protected: BTreeInternalNode() : ParentType(EMPTY_LEVEL), _validLeaves(0u) @@ -326,7 +325,7 @@ private: _validLeaves = rhs._validLeaves; return *this; } - +private: template <typename NodeAllocatorType> uint32_t countValidLeaves(uint32_t start, uint32_t end, NodeAllocatorType &allocator); @@ -435,17 +434,16 @@ public: typedef BTreeKeyData<KeyT, DataT> KeyDataType; typedef KeyT KeyType; typedef DataT DataType; -private: +protected: BTreeLeafNode() : ParentType(LEAF_LEVEL) {} -protected: BTreeLeafNode(const BTreeLeafNode &rhs) : ParentType(rhs) {} BTreeLeafNode(const KeyDataType *smallArray, uint32_t arraySize); - ~BTreeLeafNode() {} + ~BTreeLeafNode() = default; BTreeLeafNode &operator=(const BTreeLeafNode &rhs) { ParentType::operator=(rhs); @@ -530,13 +528,10 @@ extern template class BTreeNodeTT<uint32_t, int32_t, MinMaxAggregated, 16>; extern template class BTreeInternalNode<uint32_t, NoAggregated, 16>; extern template class BTreeInternalNode<uint32_t, MinMaxAggregated, 16>; extern template class BTreeLeafNode<uint32_t, uint32_t, NoAggregated, 16>; -extern template class BTreeLeafNode<uint32_t, BTreeNoLeafData, NoAggregated, - 16>; +extern template class BTreeLeafNode<uint32_t, BTreeNoLeafData, NoAggregated, 16>; extern template class BTreeLeafNode<uint32_t, int32_t, MinMaxAggregated, 16>; extern template class BTreeLeafNodeTemp<uint32_t, uint32_t, NoAggregated, 16>; -extern template class BTreeLeafNodeTemp<uint32_t, int32_t, MinMaxAggregated, - 16>; -extern template class BTreeLeafNodeTemp<uint32_t, BTreeNoLeafData, - NoAggregated, 16>; +extern template class BTreeLeafNodeTemp<uint32_t, int32_t, MinMaxAggregated, 16>; +extern template class BTreeLeafNodeTemp<uint32_t, BTreeNoLeafData, NoAggregated, 16>; } diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.cpp b/vespalib/src/vespa/vespalib/btree/btreenodestore.cpp index fd353fa6cd9..5ffce85b78c 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.cpp +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.cpp @@ -5,6 +5,7 @@ #include "btreeroot.h" #include "btreenodeallocator.h" #include <vespa/vespalib/datastore/datastore.h> +#include <vespa/vespalib/datastore/buffer_type.hpp> namespace vespalib::btree { @@ -18,4 +19,33 @@ template class BTreeNodeStore<uint32_t, int32_t, MinMaxAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; +template class BTreeNodeBufferType<BTreeInternalNode<uint32_t, NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>>; +template class BTreeNodeBufferType<BTreeInternalNode<uint32_t, MinMaxAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>>; + +template class BTreeNodeBufferType<BTreeLeafNode<uint32_t, uint32_t, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>>; +template class BTreeNodeBufferType<BTreeLeafNode<uint32_t, BTreeNoLeafData, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>>; +template class BTreeNodeBufferType<BTreeLeafNode<uint32_t, int32_t, MinMaxAggregated, BTreeDefaultTraits::LEAF_SLOTS>>; + } + +namespace vespalib::datastore { + +using namespace btree; + +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_INTERNALNODE(uint32_t, NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS); +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_INTERNALNODE(uint32_t, MinMaxAggregated, BTreeDefaultTraits::INTERNAL_SLOTS); +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_INTERNALNODE(EntryRef, NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS); + +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(uint32_t, uint32_t, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS); +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(uint32_t, BTreeNoLeafData, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS); +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(uint32_t, int32_t , MinMaxAggregated, BTreeDefaultTraits::LEAF_SLOTS); +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(EntryRef, uint32_t, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS); +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(EntryRef, BTreeNoLeafData, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS); +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(EntryRef, EntryRef, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS); + +template class BufferType<BTreeKeyData<uint32_t, uint32_t>>; +template class BufferType<BTreeKeyData<uint32_t, int32_t>>; +template class BufferType<BTreeKeyData<uint32_t, BTreeNoLeafData>>; +template class BufferType<BTreeKeyData<uint32_t, EntryRef>>; + +}
\ No newline at end of file diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.h b/vespalib/src/vespa/vespalib/btree/btreenodestore.h index c887e436e83..391229e1ffb 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.h @@ -16,24 +16,27 @@ public: } }; +template <typename ToFreeze> +struct FrozenBtreeNode : public ToFreeze { + FrozenBtreeNode() : ToFreeze() { this->freeze(); } +}; template <typename EntryType> -class BTreeNodeBufferType : public datastore::BufferType<EntryType> +class BTreeNodeBufferType : public datastore::BufferType<EntryType, FrozenBtreeNode<EntryType>> { - typedef datastore::BufferType<EntryType> ParentType; + using ParentType = datastore::BufferType<EntryType, FrozenBtreeNode<EntryType>>; using ParentType::_emptyEntry; using ParentType::_arraySize; + using ElemCount = typename ParentType::ElemCount; using CleanContext = typename ParentType::CleanContext; public: BTreeNodeBufferType(uint32_t minArrays, uint32_t maxArrays) : ParentType(1, minArrays, maxArrays) - { - _emptyEntry.freeze(); - } + { } - void initializeReservedElements(void *buffer, size_t reservedElements) override; + void initializeReservedElements(void *buffer, ElemCount reservedElements) override; - void cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext cleanCtx) override; + void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; }; @@ -219,4 +222,33 @@ extern template class BTreeNodeStore<uint32_t, int32_t, MinMaxAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; +extern template class BTreeNodeBufferType<BTreeInternalNode<uint32_t, NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>>; +extern template class BTreeNodeBufferType<BTreeInternalNode<uint32_t, MinMaxAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>>; + +extern template class BTreeNodeBufferType<BTreeLeafNode<uint32_t, uint32_t, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>>; +extern template class BTreeNodeBufferType<BTreeLeafNode<uint32_t, BTreeNoLeafData, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>>; +extern template class BTreeNodeBufferType<BTreeLeafNode<uint32_t, int32_t, MinMaxAggregated, BTreeDefaultTraits::LEAF_SLOTS>>; + } + +namespace vespalib::datastore { + +using namespace btree; + +extern template class BufferType<BTreeInternalNode<uint32_t, NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>, + FrozenBtreeNode<BTreeInternalNode<uint32_t, NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>>>; +extern template class BufferType<BTreeInternalNode<uint32_t, MinMaxAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>, + FrozenBtreeNode<BTreeInternalNode<uint32_t, MinMaxAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>>>; + +extern template class BufferType<BTreeLeafNode<uint32_t, uint32_t, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>, + FrozenBtreeNode<BTreeLeafNode<uint32_t, uint32_t, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>>>; +extern template class BufferType<BTreeLeafNode<uint32_t, BTreeNoLeafData, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>, + FrozenBtreeNode<BTreeLeafNode<uint32_t, BTreeNoLeafData, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>>>; +extern template class BufferType<BTreeLeafNode<uint32_t, int32_t, MinMaxAggregated, BTreeDefaultTraits::LEAF_SLOTS>, + FrozenBtreeNode<BTreeLeafNode<uint32_t, int32_t, MinMaxAggregated, BTreeDefaultTraits::LEAF_SLOTS>>>; + +extern template class BufferType<BTreeKeyData<uint32_t, uint32_t>>; +extern template class BufferType<BTreeKeyData<uint32_t, int32_t>>; +extern template class BufferType<BTreeKeyData<uint32_t, BTreeNoLeafData>>; + +}
\ No newline at end of file diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp index 5a3846416cc..b4ad927b618 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp @@ -9,7 +9,7 @@ namespace vespalib::btree { template <typename EntryType> void -BTreeNodeBufferType<EntryType>::initializeReservedElements(void *buffer, size_t reservedElements) +BTreeNodeBufferType<EntryType>::initializeReservedElements(void *buffer, ElemCount reservedElements) { ParentType::initializeReservedElements(buffer, reservedElements); EntryType *e = static_cast<EntryType *>(buffer); @@ -22,7 +22,7 @@ BTreeNodeBufferType<EntryType>::initializeReservedElements(void *buffer, size_t template <typename EntryType> void -BTreeNodeBufferType<EntryType>::cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext) +BTreeNodeBufferType<EntryType>::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) { EntryType *e = static_cast<EntryType *>(buffer) + offset; for (size_t j = numElems; j != 0; --j) { @@ -31,9 +31,6 @@ BTreeNodeBufferType<EntryType>::cleanHold(void *buffer, size_t offset, size_t nu } } - - - template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS> BTreeNodeStore<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: @@ -81,3 +78,10 @@ finishCompact(const std::vector<uint32_t> &toHold) } } + +#define VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_INTERNALNODE(K, A, S) \ + template class BufferType<BTreeInternalNode<K, A, S>, \ + FrozenBtreeNode<BTreeInternalNode<K, A, S>>> +#define VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(K, V, A, S) \ + template class BufferType<BTreeLeafNode<K, V, A, S>, \ + FrozenBtreeNode<BTreeLeafNode<K, V, A, S>>> diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.cpp b/vespalib/src/vespa/vespalib/btree/btreestore.cpp index 9868721da88..364064232c5 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.cpp +++ b/vespalib/src/vespa/vespalib/btree/btreestore.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "btreestore.h" #include "btreestore.hpp" #include "btreeiterator.hpp" +#include <vespa/vespalib/datastore/buffer_type.hpp> namespace vespalib::btree { @@ -11,3 +11,13 @@ template class BTreeStore<uint32_t, BTreeNoLeafData, NoAggregated, std::less<uin template class BTreeStore<uint32_t, int32_t, MinMaxAggregated, std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>; } + +namespace vespalib::datastore { + +using namespace btree; + +template class BufferType<BTreeRoot<uint32_t, uint32_t, NoAggregated, std::less<uint32_t>, BTreeDefaultTraits>>; +template class BufferType<BTreeRoot<uint32_t, BTreeNoLeafData, NoAggregated, std::less<uint32_t>, BTreeDefaultTraits>>; +template class BufferType<BTreeRoot<uint32_t, int32_t, MinMaxAggregated, std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>>; + +}
\ No newline at end of file diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.h b/vespalib/src/vespa/vespalib/btree/btreestore.h index 7889d90278a..d822c72de60 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreestore.h @@ -28,8 +28,7 @@ public: typedef DataStoreType::RefType RefType; typedef BTreeKeyData<KeyT, DataT> KeyDataType; - typedef BTreeRoot<KeyT, DataT, AggrT, CompareT, TraitsT, - AggrCalcT> BTreeType; + typedef BTreeRoot<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT> BTreeType; typedef BTreeInternalNode<KeyT, AggrT, TraitsT::INTERNAL_SLOTS> InternalNodeType; typedef BTreeLeafNode<KeyT, DataT, AggrT, TraitsT::LEAF_SLOTS> @@ -506,4 +505,12 @@ extern template class BTreeStore<uint32_t, int32_t, } +namespace vespalib::datastore { +using namespace btree; + +extern template class BufferType<BTreeRoot<uint32_t, uint32_t, NoAggregated, std::less<uint32_t>, BTreeDefaultTraits>>; +extern template class BufferType<BTreeRoot<uint32_t, BTreeNoLeafData, NoAggregated, std::less<uint32_t>, BTreeDefaultTraits>>; +extern template class BufferType<BTreeRoot<uint32_t, int32_t, MinMaxAggregated, std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>>; + +} diff --git a/vespalib/src/vespa/vespalib/btree/btreetraits.h b/vespalib/src/vespa/vespalib/btree/btreetraits.h index 6b64049899f..52a28799bb6 100644 --- a/vespalib/src/vespa/vespalib/btree/btreetraits.h +++ b/vespalib/src/vespa/vespalib/btree/btreetraits.h @@ -14,6 +14,6 @@ struct BTreeTraits { static const bool BINARY_SEEK = BS; }; -typedef BTreeTraits<16, 16, 10, true> BTreeDefaultTraits; +using BTreeDefaultTraits = BTreeTraits<16, 16, 10, true>; } diff --git a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt index b8c9dae7174..16c5d3973e8 100644 --- a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt @@ -1,12 +1,14 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vespalib_vespalib_datastore OBJECT SOURCES + array_store.cpp array_store_config.cpp buffer_type.cpp bufferstate.cpp datastore.cpp datastorebase.cpp entryref.cpp + unique_store.cpp unique_store_string_allocator.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.cpp b/vespalib/src/vespa/vespalib/datastore/array_store.cpp new file mode 100644 index 00000000000..1cfe45a1727 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/array_store.cpp @@ -0,0 +1,14 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "array_store.hpp" +#include "buffer_type.hpp" + +namespace vespalib::datastore { + +template class BufferType<vespalib::Array<uint8_t>>; +template class BufferType<vespalib::Array<uint32_t>>; +template class BufferType<vespalib::Array<int32_t>>; +template class BufferType<vespalib::Array<std::string>>; +template class BufferType<vespalib::Array<AtomicEntryRef>>; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index 05af192c77f..37f09b26205 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -7,6 +7,7 @@ #include "bufferstate.h" #include "datastore.h" #include "entryref.h" +#include "atomic_entry_ref.h" #include "i_compaction_context.h" #include <vespa/vespalib/util/array.h> @@ -40,14 +41,14 @@ private: using CleanContext = typename ParentType::CleanContext; public: LargeArrayType(const AllocSpec &spec); - virtual void cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext cleanCtx) override; + void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; }; uint32_t _largeArrayTypeId; uint32_t _maxSmallArraySize; DataStoreType _store; - std::vector<std::unique_ptr<SmallArrayType>> _smallArrayTypes; + std::vector<SmallArrayType> _smallArrayTypes; LargeArrayType _largeArrayType; using generation_t = vespalib::GenerationHandler::generation_t; @@ -122,4 +123,10 @@ public: float allocGrowFactor); }; +extern template class BufferType<vespalib::Array<uint8_t>>; +extern template class BufferType<vespalib::Array<uint32_t>>; +extern template class BufferType<vespalib::Array<int32_t>>; +extern template class BufferType<vespalib::Array<std::string>>; +extern template class BufferType<vespalib::Array<AtomicEntryRef>>; + } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp index aa085698aa9..c1fa14cac3c 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp @@ -17,7 +17,7 @@ ArrayStore<EntryT, RefT>::LargeArrayType::LargeArrayType(const AllocSpec &spec) template <typename EntryT, typename RefT> void -ArrayStore<EntryT, RefT>::LargeArrayType::cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext cleanCtx) +ArrayStore<EntryT, RefT>::LargeArrayType::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) { LargeArray *elem = static_cast<LargeArray *>(buffer) + offset; for (size_t i = 0; i < numElems; ++i) { @@ -33,13 +33,15 @@ ArrayStore<EntryT, RefT>::initArrayTypes(const ArrayStoreConfig &cfg) { _largeArrayTypeId = _store.addType(&_largeArrayType); assert(_largeArrayTypeId == 0); + _smallArrayTypes.reserve(_maxSmallArraySize); for (uint32_t arraySize = 1; arraySize <= _maxSmallArraySize; ++arraySize) { const AllocSpec &spec = cfg.specForSize(arraySize); - _smallArrayTypes.push_back(std::make_unique<SmallArrayType> - (arraySize, spec.minArraysInBuffer, spec.maxArraysInBuffer, - spec.numArraysForNewBuffer, spec.allocGrowFactor)); - uint32_t typeId = _store.addType(_smallArrayTypes.back().get()); - assert(typeId == arraySize); // Enforce 1-to-1 mapping between type ids and sizes for small arrays + _smallArrayTypes.emplace_back(arraySize, spec.minArraysInBuffer, spec.maxArraysInBuffer, + spec.numArraysForNewBuffer, spec.allocGrowFactor); + } + for (auto & type : _smallArrayTypes) { + uint32_t typeId = _store.addType(&type); + assert(typeId == type.getArraySize()); // Enforce 1-to-1 mapping between type ids and sizes for small arrays } } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp index ebc1dc5a5b7..848be2196ed 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "buffer_type.h" -#include <algorithm> +#include "atomic_entry_ref.h" +#include "buffer_type.hpp" #include <cassert> namespace vespalib::datastore { @@ -25,7 +25,7 @@ BufferTypeBase::BufferTypeBase(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays, uint32_t numArraysForNewBuffer, - float allocGrowFactor) + float allocGrowFactor) noexcept : _arraySize(arraySize), _minArrays(std::min(minArrays, maxArrays)), _maxArrays(maxArrays), @@ -41,7 +41,7 @@ BufferTypeBase::BufferTypeBase(uint32_t arraySize, BufferTypeBase::BufferTypeBase(uint32_t arraySize, uint32_t minArrays, - uint32_t maxArrays) + uint32_t maxArrays) noexcept : BufferTypeBase(arraySize, minArrays, maxArrays, 0u, DEFAULT_ALLOC_GROW_FACTOR) { } @@ -55,7 +55,7 @@ BufferTypeBase::~BufferTypeBase() assert(_lastUsedElems == nullptr); } -size_t +ElemCount BufferTypeBase::getReservedElements(uint32_t bufferId) const { return bufferId == 0 ? _arraySize : 0u; @@ -71,7 +71,7 @@ BufferTypeBase::flushLastUsed() } void -BufferTypeBase::onActive(uint32_t bufferId, size_t *usedElems, size_t &deadElems, void *buffer) +BufferTypeBase::onActive(uint32_t bufferId, ElemCount *usedElems, ElemCount &deadElems, void *buffer) { flushLastUsed(); ++_activeBuffers; @@ -85,7 +85,7 @@ BufferTypeBase::onActive(uint32_t bufferId, size_t *usedElems, size_t &deadElems } void -BufferTypeBase::onHold(const size_t *usedElems) +BufferTypeBase::onHold(const ElemCount *usedElems) { if (usedElems == _lastUsedElems) { flushLastUsed(); @@ -98,7 +98,7 @@ BufferTypeBase::onHold(const size_t *usedElems) } void -BufferTypeBase::onFree(size_t usedElems) +BufferTypeBase::onFree(ElemCount usedElems) { --_holdBuffers; assert(_holdUsedElems >= usedElems); @@ -114,7 +114,7 @@ BufferTypeBase::clampMaxArrays(uint32_t maxArrays) } size_t -BufferTypeBase::calcArraysToAlloc(uint32_t bufferId, size_t elemsNeeded, bool resizing) const +BufferTypeBase::calcArraysToAlloc(uint32_t bufferId, ElemCount elemsNeeded, bool resizing) const { size_t reservedElems = getReservedElements(bufferId); size_t usedElems = (resizing ? 0 : _activeUsedElems); @@ -138,5 +138,13 @@ BufferTypeBase::calcArraysToAlloc(uint32_t bufferId, size_t elemsNeeded, bool re return result; } +template class BufferType<char>; +template class BufferType<uint8_t>; +template class BufferType<uint32_t>; +template class BufferType<uint64_t>; +template class BufferType<int32_t>; +template class BufferType<std::string>; +template class BufferType<AtomicEntryRef>; + } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.h b/vespalib/src/vespa/vespalib/datastore/buffer_type.h index dbd399a1bcd..4dc27fb93fb 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.h @@ -2,11 +2,12 @@ #pragma once -#include <cstdint> -#include <cstddef> +#include "atomic_entry_ref.h" +#include <string> namespace vespalib::datastore { +using ElemCount = uint32_t; /** * Abstract class used to manage allocation and de-allocation of a specific data type in underlying memory buffers in a data store. * Each buffer is owned by an instance of BufferState. @@ -16,148 +17,99 @@ namespace vespalib::datastore { */ class BufferTypeBase { -protected: - uint32_t _arraySize; // Number of elements in an allocation unit - uint32_t _minArrays; // Minimum number of arrays to allocate in a buffer - uint32_t _maxArrays; // Maximum number of arrays to allocate in a buffer - // Number of arrays needed before allocating a new buffer instead of just resizing the first one - uint32_t _numArraysForNewBuffer; - float _allocGrowFactor; - uint32_t _activeBuffers; - uint32_t _holdBuffers; - size_t _activeUsedElems; // used elements in all but last active buffer - size_t _holdUsedElems; // used elements in all held buffers - const size_t *_lastUsedElems; // used elements in last active buffer - public: + using ElemCount = vespalib::datastore::ElemCount; class CleanContext { private: size_t &_extraUsedBytes; size_t &_extraHoldBytes; public: - CleanContext(size_t &extraUsedBytes, size_t &extraHoldBytes) : _extraUsedBytes(extraUsedBytes), _extraHoldBytes(extraHoldBytes) {} + CleanContext(size_t &extraUsedBytes, size_t &extraHoldBytes) + : _extraUsedBytes(extraUsedBytes), + _extraHoldBytes(extraHoldBytes) + {} void extraBytesCleaned(size_t value); }; - BufferTypeBase(const BufferTypeBase &rhs) = delete; BufferTypeBase & operator=(const BufferTypeBase &rhs) = delete; - BufferTypeBase(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays); + BufferTypeBase(BufferTypeBase &&rhs) noexcept = default; + BufferTypeBase & operator=(BufferTypeBase &&rhs) noexcept = default; + BufferTypeBase(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept; BufferTypeBase(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays, - uint32_t numArraysForNewBuffer, float allocGrowFactor); + uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept; virtual ~BufferTypeBase(); - virtual void destroyElements(void *buffer, size_t numElems) = 0; - virtual void fallbackCopy(void *newBuffer, const void *oldBuffer, size_t numElems) = 0; + virtual void destroyElements(void *buffer, ElemCount numElems) = 0; + virtual void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) = 0; // Return number of reserved elements at start of buffer, to avoid // invalid reference and handle data at negative offset (alignment // hacks) as used by dense tensor store. - virtual size_t getReservedElements(uint32_t bufferId) const; + virtual ElemCount getReservedElements(uint32_t bufferId) const; // Initialize reserved elements at start of buffer. - virtual void initializeReservedElements(void *buffer, size_t reservedElements) = 0; + virtual void initializeReservedElements(void *buffer, ElemCount reservedElements) = 0; virtual size_t elementSize() const = 0; - virtual void cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext cleanCtx) = 0; + virtual void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) = 0; size_t getArraySize() const { return _arraySize; } void flushLastUsed(); - virtual void onActive(uint32_t bufferId, size_t *usedElems, size_t &deadElems, void *buffer); - void onHold(const size_t *usedElems); - virtual void onFree(size_t usedElems); + virtual void onActive(uint32_t bufferId, ElemCount *usedElems, ElemCount &deadElems, void *buffer); + void onHold(const ElemCount *usedElems); + virtual void onFree(ElemCount usedElems); /** * Calculate number of arrays to allocate for new buffer given how many elements are needed. */ - virtual size_t calcArraysToAlloc(uint32_t bufferId, size_t elementsNeeded, bool resizing) const; + virtual size_t calcArraysToAlloc(uint32_t bufferId, ElemCount elementsNeeded, bool resizing) const; void clampMaxArrays(uint32_t maxArrays); uint32_t getActiveBuffers() const { return _activeBuffers; } size_t getMaxArrays() const { return _maxArrays; } uint32_t getNumArraysForNewBuffer() const { return _numArraysForNewBuffer; } +protected: + uint32_t _arraySize; // Number of elements in an allocation unit + uint32_t _minArrays; // Minimum number of arrays to allocate in a buffer + uint32_t _maxArrays; // Maximum number of arrays to allocate in a buffer + // Number of arrays needed before allocating a new buffer instead of just resizing the first one + uint32_t _numArraysForNewBuffer; + float _allocGrowFactor; + uint32_t _activeBuffers; + uint32_t _holdBuffers; + size_t _activeUsedElems; // used elements in all but last active buffer + size_t _holdUsedElems; // used elements in all held buffers + const ElemCount *_lastUsedElems; // used elements in last active buffer }; /** * Concrete class used to manage allocation and de-allocation of elements of type EntryType in data store buffers. */ -template <typename EntryType> +template <typename EntryType, typename EmptyType = EntryType> class BufferType : public BufferTypeBase { protected: - EntryType _emptyEntry; + static EntryType _emptyEntry; public: + BufferType() noexcept : BufferType(1,1,1) {} BufferType(const BufferType &rhs) = delete; BufferType & operator=(const BufferType &rhs) = delete; - BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays); + BufferType(BufferType && rhs) noexcept = default; + BufferType & operator=(BufferType && rhs) noexcept = default; + BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept; BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays, - uint32_t numArraysForNewBuffer, float allocGrowFactor); + uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept; ~BufferType(); - void destroyElements(void *buffer, size_t numElems) override; - void fallbackCopy(void *newBuffer, const void *oldBuffer, size_t numElems) override; - void initializeReservedElements(void *buffer, size_t reservedElements) override; - void cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext cleanCxt) override; + void destroyElements(void *buffer, ElemCount numElems) override; + void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override; + void initializeReservedElements(void *buffer, ElemCount reservedElements) override; + void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCxt) override; size_t elementSize() const override { return sizeof(EntryType); } }; -template <typename EntryType> -BufferType<EntryType>::BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) - : BufferTypeBase(arraySize, minArrays, maxArrays), - _emptyEntry() -{ } - -template <typename EntryType> -BufferType<EntryType>::BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays, - uint32_t numArraysForNewBuffer, float allocGrowFactor) - : BufferTypeBase(arraySize, minArrays, maxArrays, numArraysForNewBuffer, allocGrowFactor), - _emptyEntry() -{ } - -template <typename EntryType> -BufferType<EntryType>::~BufferType() { } - -template <typename EntryType> -void -BufferType<EntryType>::destroyElements(void *buffer, size_t numElems) -{ - EntryType *e = static_cast<EntryType *>(buffer); - for (size_t j = numElems; j != 0; --j) { - e->~EntryType(); - ++e; - } -} - -template <typename EntryType> -void -BufferType<EntryType>::fallbackCopy(void *newBuffer, - const void *oldBuffer, - size_t numElems) -{ - EntryType *d = static_cast<EntryType *>(newBuffer); - const EntryType *s = static_cast<const EntryType *>(oldBuffer); - for (size_t j = numElems; j != 0; --j) { - new (static_cast<void *>(d)) EntryType(*s); - ++s; - ++d; - } -} - -template <typename EntryType> -void -BufferType<EntryType>::initializeReservedElements(void *buffer, size_t reservedElems) -{ - EntryType *e = static_cast<EntryType *>(buffer); - for (size_t j = reservedElems; j != 0; --j) { - new (static_cast<void *>(e)) EntryType(_emptyEntry); - ++e; - } -} - -template <typename EntryType> -void -BufferType<EntryType>::cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext) -{ - EntryType *e = static_cast<EntryType *>(buffer) + offset; - for (size_t j = numElems; j != 0; --j) { - *e = _emptyEntry; - ++e; - } -} +extern template class BufferType<char>; +extern template class BufferType<uint8_t>; +extern template class BufferType<uint32_t>; +extern template class BufferType<uint64_t>; +extern template class BufferType<int32_t>; +extern template class BufferType<std::string>; +extern template class BufferType<AtomicEntryRef>; } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp new file mode 100644 index 00000000000..3f078b7ea6c --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp @@ -0,0 +1,74 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "buffer_type.h" + +namespace vespalib::datastore { + +template <typename EntryType, typename EmptyType> +BufferType<EntryType, EmptyType>::BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept + : BufferTypeBase(arraySize, minArrays, maxArrays) +{ } + +template <typename EntryType, typename EmptyType> +BufferType<EntryType, EmptyType>::BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays, + uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept + : BufferTypeBase(arraySize, minArrays, maxArrays, numArraysForNewBuffer, allocGrowFactor) +{ } + +template <typename EntryType, typename EmptyType> +BufferType<EntryType, EmptyType>::~BufferType() = default; + +template <typename EntryType, typename EmptyType> +void +BufferType<EntryType, EmptyType>::destroyElements(void *buffer, ElemCount numElems) +{ + EntryType *e = static_cast<EntryType *>(buffer); + for (size_t j = numElems; j != 0; --j) { + e->~EntryType(); + ++e; + } +} + +template <typename EntryType, typename EmptyType> +void +BufferType<EntryType, EmptyType>::fallbackCopy(void *newBuffer, + const void *oldBuffer, + ElemCount numElems) +{ + EntryType *d = static_cast<EntryType *>(newBuffer); + const EntryType *s = static_cast<const EntryType *>(oldBuffer); + for (size_t j = numElems; j != 0; --j) { + new (static_cast<void *>(d)) EntryType(*s); + ++s; + ++d; + } +} + +template <typename EntryType, typename EmptyType> +void +BufferType<EntryType, EmptyType>::initializeReservedElements(void *buffer, ElemCount reservedElems) +{ + EntryType *e = static_cast<EntryType *>(buffer); + for (size_t j = reservedElems; j != 0; --j) { + new (static_cast<void *>(e)) EntryType(_emptyEntry); + ++e; + } +} + +template <typename EntryType, typename EmptyType> +void +BufferType<EntryType, EmptyType>::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) +{ + EntryType *e = static_cast<EntryType *>(buffer) + offset; + for (size_t j = numElems; j != 0; --j) { + *e = _emptyEntry; + ++e; + } +} + +template <typename EntryType, typename EmptyType> +EntryType BufferType<EntryType, EmptyType>::_emptyEntry = EmptyType(); + +} diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp index 8db1b6ff076..2edd251452a 100644 --- a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp +++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp @@ -27,8 +27,8 @@ BufferState::BufferState() _prevHasFree(nullptr), _typeHandler(nullptr), _buffer(Alloc::alloc(0, MemoryAllocator::HUGEPAGE_SIZE)), - _typeId(0), _arraySize(0), + _typeId(0), _state(FREE), _disableElemHoldList(false), _compacting(false) @@ -122,6 +122,7 @@ BufferState::onActive(uint32_t bufferId, uint32_t typeId, _allocElems = alloc.elements; _state = ACTIVE; _typeHandler = typeHandler; + assert(typeId <= std::numeric_limits<uint16_t>::max()); _typeId = typeId; _arraySize = _typeHandler->getArraySize(); typeHandler->onActive(bufferId, &_usedElems, _deadElems, buffer); @@ -142,7 +143,7 @@ BufferState::onHold() _typeHandler->onHold(&_usedElems); if ( ! isFreeListEmpty()) { removeFromFreeListList(); - _freeList.reset(); + FreeList().swap(_freeList); } assert(_nextHasFree == nullptr); assert(_prevHasFree == nullptr); @@ -217,7 +218,7 @@ BufferState::setFreeListList(FreeListList *freeListList) if (freeListList != nullptr) { addToFreeListList(); // Changed free list list } else { - _freeList.reset(); // Free lists have been disabled + FreeList().swap(_freeList);; // Free lists have been disabled } } } @@ -239,9 +240,6 @@ BufferState::addToFreeListList() _prevHasFree = this; } _freeListList->_head = this; - if ( ! _freeList) { - _freeList = std::make_unique<FreeList>(); - } } diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.h b/vespalib/src/vespa/vespalib/datastore/bufferstate.h index 5f0a2dedbd7..a90e4ea7833 100644 --- a/vespalib/src/vespa/vespalib/datastore/bufferstate.h +++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.h @@ -36,24 +36,24 @@ public: using FreeList = vespalib::Array<EntryRef>; - enum State { + enum State : uint8_t { FREE, ACTIVE, HOLD }; private: - size_t _usedElems; - size_t _allocElems; - size_t _deadElems; - size_t _holdElems; + ElemCount _usedElems; + ElemCount _allocElems; + ElemCount _deadElems; + ElemCount _holdElems; // Number of bytes that are heap allocated by elements that are stored in this buffer. // For simple types this is 0. size_t _extraUsedBytes; // Number of bytes that are heap allocated by elements that are stored in this buffer and is now on hold. // For simple types this is 0. size_t _extraHoldBytes; - std::unique_ptr<FreeList> _freeList; + FreeList _freeList; FreeListList *_freeListList; // non-nullptr if free lists are enabled // nullptr if not on circular list of buffer states with free elems @@ -62,12 +62,11 @@ private: BufferTypeBase *_typeHandler; Alloc _buffer; - uint32_t _typeId; uint32_t _arraySize; - State _state; - bool _disableElemHoldList; - bool _compacting; - + uint16_t _typeId; + State _state : 2; + bool _disableElemHoldList : 1; + bool _compacting : 1; public: /* * TODO: Check if per-buffer free lists are useful, or if @@ -132,9 +131,9 @@ public: * Pop element from free list. */ EntryRef popFreeList() { - EntryRef ret = _freeList->back(); - _freeList->pop_back(); - if (_freeList->empty()) { + EntryRef ret = _freeList.back(); + _freeList.pop_back(); + if (isFreeListEmpty()) { removeFromFreeListList(); } _deadElems -= _arraySize; @@ -148,7 +147,7 @@ public: _usedElems += numElems; _extraUsedBytes += extraBytes; } - void cleanHold(void *buffer, size_t offset, size_t numElems) { + void cleanHold(void *buffer, size_t offset, ElemCount numElems) { _typeHandler->cleanHold(buffer, offset, numElems, BufferTypeBase::CleanContext(_extraUsedBytes, _extraHoldBytes)); } void dropBuffer(void *&buffer); @@ -184,8 +183,8 @@ public: } bool hasDisabledElemHoldList() const { return _disableElemHoldList; } - bool isFreeListEmpty() const { return !_freeList || _freeList->empty();} - FreeList &freeList() { return *_freeList; } + bool isFreeListEmpty() const { return _freeList.empty();} + FreeList &freeList() { return _freeList; } const FreeListList *freeListList() const { return _freeListList; } FreeListList *freeListList() { return _freeListList; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.cpp b/vespalib/src/vespa/vespalib/datastore/unique_store.cpp new file mode 100644 index 00000000000..1cdff0e2356 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.cpp @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "unique_store.hpp" +#include "buffer_type.hpp" + +namespace vespalib::datastore { + +template class BufferType<UniqueStoreEntry<int8_t>>; +template class BufferType<UniqueStoreEntry<int16_t>>; +template class BufferType<UniqueStoreEntry<int32_t>>; +template class BufferType<UniqueStoreEntry<int64_t>>; +template class BufferType<UniqueStoreEntry<uint32_t>>; +template class BufferType<UniqueStoreEntry<float>>; +template class BufferType<UniqueStoreEntry<double>>; + +using namespace btree; + +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_INTERNALNODE(EntryRef, NoAggregated, uniquestore::DefaultDictionaryTraits::INTERNAL_SLOTS); +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(EntryRef, BTreeNoLeafData, NoAggregated, uniquestore::DefaultDictionaryTraits::LEAF_SLOTS); + +} diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp index 4f8b1b878f0..b828db081ba 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "unique_store_string_allocator.hpp" +#include "buffer_type.hpp" namespace vespalib::datastore { @@ -39,14 +40,14 @@ UniqueStoreSmallStringBufferType::UniqueStoreSmallStringBufferType(uint32_t arra UniqueStoreSmallStringBufferType::~UniqueStoreSmallStringBufferType() = default; void -UniqueStoreSmallStringBufferType::destroyElements(void *, size_t) +UniqueStoreSmallStringBufferType::destroyElements(void *, ElemCount) { static_assert(std::is_trivially_destructible<UniqueStoreSmallStringEntry>::value, "UniqueStoreSmallStringEntry must be trivially destructable"); } void -UniqueStoreSmallStringBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, size_t numElems) +UniqueStoreSmallStringBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) { static_assert(std::is_trivially_copyable<UniqueStoreSmallStringEntry>::value, "UniqueStoreSmallStringEntry must be trivially copyable"); @@ -54,7 +55,7 @@ UniqueStoreSmallStringBufferType::fallbackCopy(void *newBuffer, const void *oldB } void -UniqueStoreSmallStringBufferType::cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext) +UniqueStoreSmallStringBufferType::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) { void *e = static_cast<char *>(buffer) + offset; void *e_end = static_cast<char *>(e) + numElems; @@ -74,7 +75,7 @@ UniqueStoreExternalStringBufferType::UniqueStoreExternalStringBufferType(uint32_ UniqueStoreExternalStringBufferType::~UniqueStoreExternalStringBufferType() = default; void -UniqueStoreExternalStringBufferType::cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext cleanCtx) +UniqueStoreExternalStringBufferType::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) { UniqueStoreEntry<std::string> *elem = static_cast<UniqueStoreEntry<std::string> *>(buffer) + offset; for (size_t i = 0; i < numElems; ++i) { @@ -85,5 +86,6 @@ UniqueStoreExternalStringBufferType::cleanHold(void *buffer, size_t offset, size } template class UniqueStoreStringAllocator<EntryRefT<22>>; +template class BufferType<UniqueStoreEntry<std::string>>; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h index 7ad4384bece..43597539f12 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h @@ -59,9 +59,9 @@ class UniqueStoreSmallStringBufferType : public BufferType<char> { public: UniqueStoreSmallStringBufferType(uint32_t array_size, uint32_t max_arrays); ~UniqueStoreSmallStringBufferType() override; - void destroyElements(void *, size_t) override; - void fallbackCopy(void *newBuffer, const void *oldBuffer, size_t numElems) override; - void cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext) override; + void destroyElements(void *, ElemCount) override; + void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override; + void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) override; }; /* @@ -71,7 +71,7 @@ class UniqueStoreExternalStringBufferType : public BufferType<UniqueStoreEntry<s public: UniqueStoreExternalStringBufferType(uint32_t array_size, uint32_t max_arrays); ~UniqueStoreExternalStringBufferType() override; - void cleanHold(void *buffer, size_t offset, size_t numElems, CleanContext cleanCtx) override; + void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; }; /** @@ -130,4 +130,6 @@ public: const DataStoreType& get_data_store() const { return _store; } }; +extern template class BufferType<UniqueStoreEntry<std::string> >; + } |