diff options
25 files changed, 801 insertions, 56 deletions
diff --git a/client/go/internal/cli/cmd/document_test.go b/client/go/internal/cli/cmd/document_test.go index 6e671190959..bce81da91c5 100644 --- a/client/go/internal/cli/cmd/document_test.go +++ b/client/go/internal/cli/cmd/document_test.go @@ -129,7 +129,7 @@ func assertDocumentSend(arguments []string, expectedOperation string, expectedMe } } if verbose { - expectedCurl := "curl -X " + expectedMethod + " -H 'Content-Type: application/json; charset=utf-8' -H 'User-Agent: Vespa CLI/0.0.0-devel'" + expectedCurl := "curl -X " + expectedMethod + " -H 'Content-Type: application/json; charset=utf-8'" if expectedPayloadFile != "" { expectedCurl += " --data-binary @" + expectedPayloadFile } diff --git a/client/go/internal/util/http.go b/client/go/internal/util/http.go index 35e35b16720..26e7937028e 100644 --- a/client/go/internal/util/http.go +++ b/client/go/internal/util/http.go @@ -19,20 +19,17 @@ type HTTPClient interface { } type defaultHTTPClient struct { - client *http.Client - setUserAgent bool + client *http.Client } func (c *defaultHTTPClient) Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) { if c.client.Timeout != timeout { // Set wanted timeout c.client.Timeout = timeout } - if c.setUserAgent { - if request.Header == nil { - request.Header = make(http.Header) - } - request.Header.Set("User-Agent", fmt.Sprintf("Vespa CLI/%s", build.Version)) + if request.Header == nil { + request.Header = make(http.Header) } + request.Header.Set("User-Agent", fmt.Sprintf("Vespa CLI/%s", build.Version)) return c.client.Do(request) } @@ -68,7 +65,6 @@ func ForceHTTP2(client HTTPClient, certificates []tls.Certificate, caCertificate if !ok { return } - c.setUserAgent = false // Let caller control all request headers var dialFunc func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) if certificates == nil { // No certificate, so force H2C (HTTP/2 over clear-text) by using a non-TLS Dialer @@ -95,6 +91,5 @@ func CreateClient(timeout time.Duration) HTTPClient { Timeout: timeout, Transport: http.DefaultTransport, }, - setUserAgent: true, } } diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index 986659773f1..d6e7745e6b1 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -17,7 +17,6 @@ import ( "github.com/go-json-experiment/json" "github.com/klauspost/compress/gzip" - "github.com/vespa-engine/vespa/client/go/internal/build" "github.com/vespa-engine/vespa/client/go/internal/util" ) @@ -29,18 +28,6 @@ const ( CompressionGzip ) -var ( - defaultHeaders http.Header = map[string][]string{ - "User-Agent": {fmt.Sprintf("Vespa CLI/%s", build.Version)}, - "Content-Type": {"application/json; charset=utf-8"}, - } - gzipHeaders http.Header = map[string][]string{ - "User-Agent": {fmt.Sprintf("Vespa CLI/%s", build.Version)}, - "Content-Type": {"application/json; charset=utf-8"}, - "Content-Encoding": {"gzip"}, - } -) - // Client represents a HTTP client for the /document/v1/ API. type Client struct { options ClientOptions @@ -233,10 +220,9 @@ func newRequest(method, url string, body io.Reader, gzipped bool) (*http.Request if err != nil { return nil, err } + req.Header.Set("Content-Type", "application/json; charset=utf-8") if gzipped { - req.Header = gzipHeaders - } else { - req.Header = defaultHeaders + req.Header.Set("Content-Encoding", "gzip") } return req, nil } diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index 30bd8406f45..c797ba5607f 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -3,6 +3,7 @@ package document import ( "bytes" "fmt" + "net/http" "reflect" "strings" "testing" @@ -112,8 +113,11 @@ func TestClientSend(t *testing.T) { if r.Method != tt.method { t.Errorf("got r.Method = %q, want %q", r.Method, tt.method) } - if !reflect.DeepEqual(r.Header, defaultHeaders) { - t.Errorf("got r.Header = %v, want %v", r.Header, defaultHeaders) + var headers http.Header = map[string][]string{ + "Content-Type": {"application/json; charset=utf-8"}, + } + if !reflect.DeepEqual(r.Header, headers) { + t.Errorf("got r.Header = %v, want %v", r.Header, headers) } if r.URL.String() != tt.url { t.Errorf("got r.URL = %q, want %q", r.URL, tt.url) diff --git a/component/src/main/java/com/yahoo/component/provider/ListenableFreezableClass.java b/component/src/main/java/com/yahoo/component/provider/ListenableFreezableClass.java index be320c9cee2..c493550e2be 100644 --- a/component/src/main/java/com/yahoo/component/provider/ListenableFreezableClass.java +++ b/component/src/main/java/com/yahoo/component/provider/ListenableFreezableClass.java @@ -29,13 +29,13 @@ public class ListenableFreezableClass extends FreezableClass implements Listenab /** Adds a listener which will be invoked when this has become frozen. */ @Override public void addFreezeListener(Runnable runnable, Executor executor) { - executionList.add(runnable,executor); + executionList.add(runnable, executor); } /** Clones this. The clone is <i>not</i> frozen and has no listeners. */ @Override public ListenableFreezableClass clone() { - ListenableFreezableClass clone=(ListenableFreezableClass)super.clone(); + ListenableFreezableClass clone = (ListenableFreezableClass)super.clone(); clone.executionList = new ExecutionList(); return clone; } diff --git a/container-core/src/main/java/com/yahoo/processing/response/Ordered.java b/container-core/src/main/java/com/yahoo/processing/response/Ordered.java index bcf110bfb67..f9c4b779a4e 100644 --- a/container-core/src/main/java/com/yahoo/processing/response/Ordered.java +++ b/container-core/src/main/java/com/yahoo/processing/response/Ordered.java @@ -8,11 +8,10 @@ package com.yahoo.processing.response; * in which it completes rather than in the order in which it is added to the list. * * @author bratseth - * @since 5.1.19 */ public interface Ordered { /** Returns false if the data in this list can be returned in any order. Default: true, meaning the order matters */ - public boolean isOrdered(); + boolean isOrdered(); } diff --git a/container-core/src/main/java/com/yahoo/processing/response/Streamed.java b/container-core/src/main/java/com/yahoo/processing/response/Streamed.java index 114e8e1414c..ca697af9bcd 100644 --- a/container-core/src/main/java/com/yahoo/processing/response/Streamed.java +++ b/container-core/src/main/java/com/yahoo/processing/response/Streamed.java @@ -8,7 +8,6 @@ package com.yahoo.processing.response; * must be deferred until the list is complete. * * @author bratseth - * @since 5.1.19 */ public interface Streamed { @@ -16,6 +15,6 @@ public interface Streamed { * Returns false if the data in this list can not be returned until it is completed. * Default: true, meaning eager streaming of the data is permissible. */ - public boolean isStreamed(); + boolean isStreamed(); } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/RenderingExecutorFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/RenderingExecutorFactory.java index f67db059470..b4e1bd9a0c4 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/RenderingExecutorFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/RenderingExecutorFactory.java @@ -18,7 +18,7 @@ class RenderingExecutorFactory { private final int availableProcessors; public RenderingExecutorFactory() { - this.maxQueuedRenderingTasksPerProcessor = 100; + this.maxQueuedRenderingTasksPerProcessor = 500; this.availableProcessors = Runtime.getRuntime().availableProcessors(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index b2af0c57c46..28f99b02e60 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -489,8 +489,7 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { } private static Agent agent(HttpRequest request) { - // TODO(mpolden): Return node-admin agent here when serialization change has rolled out everywhere - return Agent.operator; + return "node-admin".equalsIgnoreCase(request.getHeader("User-Agent")) ? Agent.nodeAdmin : Agent.operator; } private static void toSlime(Load load, Cursor object) { diff --git a/searchlib/src/vespa/searchlib/aggregation/groupinglevel.cpp b/searchlib/src/vespa/searchlib/aggregation/groupinglevel.cpp index f9b6f8c9d95..cd014360cb6 100644 --- a/searchlib/src/vespa/searchlib/aggregation/groupinglevel.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/groupinglevel.cpp @@ -17,6 +17,7 @@ GroupingLevel::GroupingLevel() noexcept _precision(-1), _isOrdered(false), _frozen(false), + _currentIndex(), _classify(), _collect(), _grouper(nullptr) @@ -87,6 +88,7 @@ GroupingLevel::MultiValueGrouper::groupDoc(Group & g, const ResultNode & result, const ResultNodeVector & rv(static_cast<const ResultNodeVector &>(result)); for (size_t i(0), m(rv.size()); i < m; i++) { const ResultNode & sr(rv.get(i)); + _currentIndex->set(i); SingleValueGrouper::groupDoc(g, sr, doc, rank); } } @@ -97,7 +99,7 @@ GroupingLevel::prepare(const Grouping * grouping, uint32_t level, bool isOrdered _isOrdered = isOrdered_; _frozen = level < grouping->getFirstLevel(); if (_classify.getResult()->inherits(ResultNodeVector::classId)) { - _grouper.reset(new MultiValueGrouper(grouping, level)); + _grouper.reset(new MultiValueGrouper(&_currentIndex, grouping, level)); } else { _grouper.reset(new SingleValueGrouper(grouping, level)); } diff --git a/searchlib/src/vespa/searchlib/aggregation/groupinglevel.h b/searchlib/src/vespa/searchlib/aggregation/groupinglevel.h index f284153b685..d3d09aff97a 100644 --- a/searchlib/src/vespa/searchlib/aggregation/groupinglevel.h +++ b/searchlib/src/vespa/searchlib/aggregation/groupinglevel.h @@ -3,6 +3,7 @@ #include "group.h" #include <vespa/searchlib/expression/aggregationrefnode.h> +#include <vespa/searchlib/expression/currentindex.h> namespace search::aggregation { @@ -20,6 +21,7 @@ private: using ResultNode = expression::ResultNode; using ExpressionNode = expression::ExpressionNode; using ExpressionTree = expression::ExpressionTree; + using CurrentIndex = expression::CurrentIndex; class Grouper { public: virtual ~Grouper() = default; @@ -53,7 +55,12 @@ private: }; class MultiValueGrouper : public SingleValueGrouper { public: - MultiValueGrouper(const Grouping * grouping, uint32_t level) noexcept : SingleValueGrouper(grouping, level) { } + MultiValueGrouper(CurrentIndex * currentIndex, const Grouping * grouping, uint32_t level) noexcept + : SingleValueGrouper(grouping, level), + _currentIndex(currentIndex) + { } + MultiValueGrouper(const MultiValueGrouper &) = default; //TODO Try to remove + MultiValueGrouper & operator=(const MultiValueGrouper &) = delete; private: template<typename Doc> void groupDoc(Group & group, const ResultNode & result, const Doc & doc, HitRank rank) const; @@ -64,11 +71,13 @@ private: groupDoc(g, result, doc, rank); } MultiValueGrouper * clone() const override { return new MultiValueGrouper(*this); } + CurrentIndex *_currentIndex; }; int64_t _maxGroups; int64_t _precision; bool _isOrdered; bool _frozen; + CurrentIndex _currentIndex; ExpressionTree _classify; Group _collect; diff --git a/searchlib/src/vespa/searchlib/expression/expressiontree.cpp b/searchlib/src/vespa/searchlib/expression/expressiontree.cpp index 5592f2f863b..cef596ca3be 100644 --- a/searchlib/src/vespa/searchlib/expression/expressiontree.cpp +++ b/searchlib/src/vespa/searchlib/expression/expressiontree.cpp @@ -4,7 +4,6 @@ #include "documentaccessornode.h" #include "relevancenode.h" #include "interpolatedlookupfunctionnode.h" -#include "arrayatlookupfunctionnode.h" #include "attributenode.h" namespace search::expression { @@ -28,8 +27,7 @@ ExpressionTree::ExpressionTree() : _attributeNodes(), _documentAccessorNodes(), _relevanceNodes(), - _interpolatedLookupNodes(), - _arrayAtLookupNodes() + _interpolatedLookupNodes() { prepare(false); } @@ -39,8 +37,7 @@ ExpressionTree::ExpressionTree(const ExpressionNode &root) : _attributeNodes(), _documentAccessorNodes(), _relevanceNodes(), - _interpolatedLookupNodes(), - _arrayAtLookupNodes() + _interpolatedLookupNodes() { prepare(false); } @@ -84,7 +81,6 @@ ExpressionTree::onPrepare(bool preserveAccurateTypes) gather(_documentAccessorNodes).from(*_root); gather(_relevanceNodes).from(*_root); gather(_interpolatedLookupNodes).from(*_root); - gather(_arrayAtLookupNodes).from(*_root); } } @@ -94,8 +90,7 @@ ExpressionTree::ExpressionTree(ExpressionNode::UP root) : _attributeNodes(), _documentAccessorNodes(), _relevanceNodes(), - _interpolatedLookupNodes(), - _arrayAtLookupNodes() + _interpolatedLookupNodes() { prepare(false); } @@ -106,8 +101,7 @@ ExpressionTree::ExpressionTree(const ExpressionTree & rhs) : _attributeNodes(), _documentAccessorNodes(), _relevanceNodes(), - _interpolatedLookupNodes(), - _arrayAtLookupNodes() + _interpolatedLookupNodes() { prepare(false); } @@ -138,7 +132,6 @@ ExpressionTree::swap(ExpressionTree & e) _documentAccessorNodes.swap(e._documentAccessorNodes); _relevanceNodes.swap(e._relevanceNodes); _interpolatedLookupNodes.swap(e._interpolatedLookupNodes); - _arrayAtLookupNodes.swap(_arrayAtLookupNodes); } ExpressionTree::~ExpressionTree() = default; @@ -160,9 +153,6 @@ struct DocIdSetter { void operator() (InterpolatedLookup *node) { node->setDocId(_docId); } - void operator() (ArrayAtLookup *node) { - node->setDocId(_docId); - } void operator() (AttributeNode *node) { node->setDocId(_docId); } @@ -186,7 +176,6 @@ ExpressionTree::execute(DocId docId, HitRank rank) const std::for_each(_attributeNodes.cbegin(), _attributeNodes.cend(), setDocId); std::for_each(_relevanceNodes.cbegin(), _relevanceNodes.cend(), setHitRank); std::for_each(_interpolatedLookupNodes.cbegin(), _interpolatedLookupNodes.cend(), setDocId); - std::for_each(_arrayAtLookupNodes.cbegin(), _arrayAtLookupNodes.cend(), setDocId); return _root->execute(); } diff --git a/searchlib/src/vespa/searchlib/expression/expressiontree.h b/searchlib/src/vespa/searchlib/expression/expressiontree.h index b9517fc5f69..54600e7fcf5 100644 --- a/searchlib/src/vespa/searchlib/expression/expressiontree.h +++ b/searchlib/src/vespa/searchlib/expression/expressiontree.h @@ -68,14 +68,12 @@ private: using DocumentAccessorNodeList = std::vector<DocumentAccessorNode *>; using RelevanceNodeList = std::vector<RelevanceNode *>; using InterpolatedLookupList = std::vector<InterpolatedLookup *>; - using ArrayAtLookupList = std::vector<ArrayAtLookup *>; ExpressionNode::CP _root; AttributeNodeList _attributeNodes; DocumentAccessorNodeList _documentAccessorNodes; RelevanceNodeList _relevanceNodes; InterpolatedLookupList _interpolatedLookupNodes; - ArrayAtLookupList _arrayAtLookupNodes; }; } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 6d19988b96b..d2892ee4429 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -65,10 +65,12 @@ vespa_define_module( src/tests/data/smart_buffer src/tests/datastore/array_store src/tests/datastore/array_store_config + src/tests/datastore/array_store_dynamic_type_mapper src/tests/datastore/buffer_stats src/tests/datastore/buffer_type src/tests/datastore/compact_buffer_candidates src/tests/datastore/datastore + src/tests/datastore/dynamic_array_buffer_type src/tests/datastore/fixed_size_hash_map src/tests/datastore/free_list src/tests/datastore/sharded_hash_map diff --git a/vespalib/src/tests/datastore/array_store_dynamic_type_mapper/CMakeLists.txt b/vespalib/src/tests/datastore/array_store_dynamic_type_mapper/CMakeLists.txt new file mode 100644 index 00000000000..d6b474a526a --- /dev/null +++ b/vespalib/src/tests/datastore/array_store_dynamic_type_mapper/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_array_store_dynamic_type_mapper_test_app TEST + SOURCES + array_store_dynamic_type_mapper_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_array_store_dynamic_type_mapper_test_app COMMAND vespalib_array_store_dynamic_type_mapper_test_app) diff --git a/vespalib/src/tests/datastore/array_store_dynamic_type_mapper/array_store_dynamic_type_mapper_test.cpp b/vespalib/src/tests/datastore/array_store_dynamic_type_mapper/array_store_dynamic_type_mapper_test.cpp new file mode 100644 index 00000000000..86b80aaa695 --- /dev/null +++ b/vespalib/src/tests/datastore/array_store_dynamic_type_mapper/array_store_dynamic_type_mapper_test.cpp @@ -0,0 +1,169 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/datastore/array_store_dynamic_type_mapper.h> +#include <vespa/vespalib/gtest/gtest.h> + +using vespalib::datastore::ArrayStoreDynamicTypeMapper; + +constexpr double default_grow_factor = 1.03; + +template <typename ElemT> +class TestBase : public testing::Test +{ +protected: + ArrayStoreDynamicTypeMapper<ElemT> _mapper; + TestBase(); + ~TestBase() override; + std::vector<size_t> get_array_sizes(uint32_t num_array_sizes); + std::vector<size_t> get_entry_sizes(uint32_t num_entry_sizes); + std::vector<size_t> get_large_array_sizes(uint32_t num_large_arrays); + void select_type_ids(std::vector<size_t> array_sizes); + void setup_mapper(uint32_t max_buffer_type_id, double grow_factor); + static uint32_t calc_max_buffer_type_id(double grow_factor); +}; + +template <typename ElemT> +TestBase<ElemT>::TestBase() + : testing::Test(), + _mapper(5, default_grow_factor) +{ +} + +template <typename ElemT> +TestBase<ElemT>::~TestBase() = default; + +template <typename ElemT> +void +TestBase<ElemT>::setup_mapper(uint32_t max_buffer_type_id, double grow_factor) +{ + _mapper = ArrayStoreDynamicTypeMapper<ElemT>(max_buffer_type_id, grow_factor); +} + +template <typename ElemT> +std::vector<size_t> +TestBase<ElemT>::get_array_sizes(uint32_t num_array_sizes) +{ + std::vector<size_t> array_sizes; + for (uint32_t type_id = 1; type_id <= num_array_sizes; ++type_id) { + array_sizes.emplace_back(_mapper.get_array_size(type_id)); + } + return array_sizes; +} + +template <typename ElemT> +std::vector<size_t> +TestBase<ElemT>::get_entry_sizes(uint32_t num_entry_sizes) +{ + std::vector<size_t> entry_sizes; + for (uint32_t type_id = 1; type_id <= num_entry_sizes; ++type_id) { + entry_sizes.emplace_back(_mapper.get_entry_size(type_id)); + } + return entry_sizes; +} + +template <typename ElemT> +std::vector<size_t> +TestBase<ElemT>::get_large_array_sizes(uint32_t num_large_array_sizes) +{ + setup_mapper(num_large_array_sizes * 100, default_grow_factor); + std::vector<size_t> result; + for (uint32_t i = 0; i < num_large_array_sizes; ++i) { + uint32_t type_id = (i + 1) * 100; + auto array_size = _mapper.get_array_size(type_id); + result.emplace_back(array_size); + EXPECT_EQ(type_id, _mapper.get_type_id(array_size)); + EXPECT_EQ(type_id, _mapper.get_type_id(array_size - 1)); + if (i + 1 == num_large_array_sizes) { + EXPECT_EQ(0u, _mapper.get_type_id(array_size + 1)); + } else { + EXPECT_EQ(type_id + 1, _mapper.get_type_id(array_size + 1)); + } + } + return result; +} + +template <typename ElemT> +void +TestBase<ElemT>::select_type_ids(std::vector<size_t> array_sizes) +{ + uint32_t type_id = 0; + std::optional<size_t> prev_array_size; + for (auto array_size : array_sizes) { + ++type_id; + EXPECT_EQ(type_id, _mapper.get_type_id(array_size)); + if (!prev_array_size.has_value() || prev_array_size.value() < array_size - 1) { + EXPECT_EQ(type_id, _mapper.get_type_id(array_size - 1)); + } else { + EXPECT_EQ(type_id - 1, _mapper.get_type_id(array_size - 1)); + } + prev_array_size = array_size; + if (array_size == array_sizes.back()) { + // Fallback to indirect storage, using type id 0 + EXPECT_EQ(0u, _mapper.get_type_id(array_size + 1)); + } else { + EXPECT_EQ(type_id + 1, _mapper.get_type_id(array_size + 1)); + } + } +} + +template <typename ElemT> +uint32_t +TestBase<ElemT>::calc_max_buffer_type_id(double grow_factor) +{ + ArrayStoreDynamicTypeMapper<ElemT> mapper(1000, grow_factor); + return mapper.get_max_small_array_type_id(1000); +} + +using ArrayStoreDynamicTypeMapperCharTest = TestBase<char>; + +TEST_F(ArrayStoreDynamicTypeMapperCharTest, array_sizes_are_calculated) +{ + EXPECT_EQ((std::vector<size_t>{1, 2, 3, 4, 5}), get_array_sizes(5)); + EXPECT_EQ((std::vector<size_t>{1, 2, 3, 4, 5}), get_entry_sizes(5)); + setup_mapper(10, 1.4); + EXPECT_EQ((std::vector<size_t>{1, 2, 3, 4, 5, 8, 12, 16, 24, 36}), get_array_sizes(10)); + EXPECT_EQ((std::vector<size_t>{1, 2, 3, 4, 5, 12, 16, 20, 28, 40}), get_entry_sizes(10)); +} + +TEST_F(ArrayStoreDynamicTypeMapperCharTest, type_ids_are_selected) +{ + select_type_ids({1, 2, 3, 4, 5}); + setup_mapper(10, 1.4); + select_type_ids({1, 2, 3, 4, 5, 8, 12, 16, 24, 36}); +} + +TEST_F(ArrayStoreDynamicTypeMapperCharTest, large_arrays_grows_exponentially) +{ + EXPECT_EQ((std::vector<size_t>{232, 5024, 97100, 1866776}), get_large_array_sizes(4)); +} + +TEST_F(ArrayStoreDynamicTypeMapperCharTest, avoid_entry_size_overflow) +{ + EXPECT_EQ(32, calc_max_buffer_type_id(2.0)); + EXPECT_EQ(410, calc_max_buffer_type_id(1.05)); + EXPECT_EQ(507, calc_max_buffer_type_id(1.04)); + EXPECT_EQ(661, calc_max_buffer_type_id(1.03)); + EXPECT_EQ(968, calc_max_buffer_type_id(1.02)); +} + +using ArrayStoreDynamicTypeMapperInt32Test = TestBase<int32_t>; + +TEST_F(ArrayStoreDynamicTypeMapperInt32Test, array_sizes_are_calculated) +{ + EXPECT_EQ((std::vector<size_t>{1, 2, 3, 4, 5}), get_array_sizes(5)); + EXPECT_EQ((std::vector<size_t>{4, 8, 12, 16, 20}), get_entry_sizes(5)); + setup_mapper(10, 1.4); + EXPECT_EQ((std::vector<size_t>{1, 2, 3, 4, 5, 7, 9, 12, 16, 22}), get_array_sizes(10)); + EXPECT_EQ((std::vector<size_t>{4, 8, 12, 16, 20, 32, 40, 52, 68, 92}), get_entry_sizes(10)); +} + +TEST_F(ArrayStoreDynamicTypeMapperInt32Test, avoid_entry_size_overflow) +{ + EXPECT_EQ(30, calc_max_buffer_type_id(2.0)); + EXPECT_EQ(395, calc_max_buffer_type_id(1.05)); + EXPECT_EQ(487, calc_max_buffer_type_id(1.04)); + EXPECT_EQ(636, calc_max_buffer_type_id(1.03)); + EXPECT_EQ(930, calc_max_buffer_type_id(1.02)); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/datastore/dynamic_array_buffer_type/CMakeLists.txt b/vespalib/src/tests/datastore/dynamic_array_buffer_type/CMakeLists.txt new file mode 100644 index 00000000000..02cb0464e7c --- /dev/null +++ b/vespalib/src/tests/datastore/dynamic_array_buffer_type/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_dynamic_array_buffer_type_test_app TEST + SOURCES + dynamic_array_buffer_type_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_dynamic_array_buffer_type_test_app COMMAND vespalib_dynamic_array_buffer_type_test_app) diff --git a/vespalib/src/tests/datastore/dynamic_array_buffer_type/dynamic_array_buffer_type_test.cpp b/vespalib/src/tests/datastore/dynamic_array_buffer_type/dynamic_array_buffer_type_test.cpp new file mode 100644 index 00000000000..d5244ae56aa --- /dev/null +++ b/vespalib/src/tests/datastore/dynamic_array_buffer_type/dynamic_array_buffer_type_test.cpp @@ -0,0 +1,249 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/datastore/dynamic_array_buffer_type.hpp> +#include <vespa/vespalib/gtest/gtest.h> +#include <ostream> + +using vespalib::datastore::BufferTypeBase; +using vespalib::datastore::DynamicArrayBufferType; +using vespalib::datastore::EntryCount; + +namespace { + +struct CleanContextBase +{ + std::atomic<size_t> _extra_used_bytes; + std::atomic<size_t> _extra_hold_bytes; + CleanContextBase() + : _extra_used_bytes(0), + _extra_hold_bytes(0) + { + } +}; + +struct MyCleanContext : public CleanContextBase, + public BufferTypeBase::CleanContext +{ + MyCleanContext() + : CleanContextBase(), + BufferTypeBase::CleanContext(_extra_used_bytes, _extra_hold_bytes) + { + } +}; + +struct Counts { + uint32_t _def_constructs; + uint32_t _value_constructs; + uint32_t _copy_constructs; + uint32_t _destructs; + uint32_t _assigns; + + Counts(uint32_t def_constructs, uint32_t value_constructs, uint32_t copy_constructs, uint32_t destructs, uint32_t assigns) + : _def_constructs(def_constructs), + _value_constructs(value_constructs), + _copy_constructs(copy_constructs), + _destructs(destructs), + _assigns(assigns) + { + } + + Counts() + : Counts(0, 0, 0, 0, 0) + { + } + bool operator==(const Counts &rhs) const { + return _def_constructs == rhs._def_constructs && + _value_constructs == rhs._value_constructs && + _copy_constructs == rhs._copy_constructs && + _destructs == rhs._destructs && + _assigns == rhs._assigns; + } +}; + +Counts counts; + +std::ostream& operator<<(std::ostream& os, const Counts& c) { + os << "{def_constructs=" << c._def_constructs << + ", value_constructs=" << c._value_constructs << + ", copy_constructs=" << c._copy_constructs << + ", destructs=" << c._destructs << + ", assigns=" << c._assigns << "}"; + return os; +} + +struct WrapInt32 { + int32_t _v; + + WrapInt32() + : _v(0) + { + ++counts._def_constructs; + } + WrapInt32(int v) + : _v(v) + { + ++counts._value_constructs; + } + WrapInt32(const WrapInt32& rhs) + : _v(rhs._v) + { + ++counts._copy_constructs; + } + WrapInt32& operator=(const WrapInt32& rhs) { + _v = rhs._v; + ++counts._assigns; + return *this; + } + ~WrapInt32() { + ++counts._destructs; + } +}; + +} + +class DynamicArrayBufferTypeTest : public testing::Test +{ +protected: + DynamicArrayBufferTypeTest(); + ~DynamicArrayBufferTypeTest() override; + + using BufferType = DynamicArrayBufferType<WrapInt32>; + + template <typename ElemT> + uint32_t get_entry_size(uint32_t array_size); + + std::vector<int> get_vector(const void *buffer, uint32_t offset, uint32_t array_size); + std::vector<int> get_vector(const void *buffer, uint32_t offset); + std::vector<int> get_max_vector(const void *buffer, uint32_t offset); + void write_entry1(); + + BufferType _buffer_type; + size_t _entry_size; + size_t _buf_size; + std::unique_ptr<char[]> _buf; +}; + +DynamicArrayBufferTypeTest::DynamicArrayBufferTypeTest() + : testing::Test(), + _buffer_type(3, 0, 10, 0, 0.2), + _entry_size(_buffer_type.entry_size()), + _buf_size(2 * _entry_size), + _buf(std::make_unique<char[]>(_buf_size)) +{ + // Call initialize_reserved_entries to force construction of empty element + _buffer_type.initialize_reserved_entries(_buf.get(), 1); + memset(_buf.get(), 55, _buf_size); + // Reset counts after empty element has been constructed + counts = Counts(); +} + +DynamicArrayBufferTypeTest::~DynamicArrayBufferTypeTest() = default; + +template <typename ElemT> +uint32_t +DynamicArrayBufferTypeTest::get_entry_size(uint32_t array_size) +{ + DynamicArrayBufferType<ElemT> my_buffer_type(array_size, 0, 10, 0, 0.2); + return my_buffer_type.entry_size(); +} + +std::vector<int> +DynamicArrayBufferTypeTest::get_vector(const void* buffer, uint32_t offset, uint32_t array_size) +{ + auto e = BufferType::get_entry(buffer, offset, _entry_size); + std::vector<int> result; + for (uint32_t i = 0; i < array_size; ++i) { + result.emplace_back(e[i]._v); + } + return result; +} + +std::vector<int> +DynamicArrayBufferTypeTest::get_vector(const void* buffer, uint32_t offset) +{ + auto e = BufferType::get_entry(buffer, offset, _entry_size); + auto array_size = BufferType::get_dynamic_array_size(e, _entry_size); + EXPECT_GE(_buffer_type.getArraySize(), array_size); + return get_vector(buffer, offset, array_size); +} + +std::vector<int> +DynamicArrayBufferTypeTest::get_max_vector(const void* buffer, uint32_t offset) +{ + auto array_size = _buffer_type.getArraySize(); + return get_vector(buffer, offset, array_size); +} + +void +DynamicArrayBufferTypeTest::write_entry1() +{ + auto e1 = BufferType::get_entry(_buf.get(), 1, _entry_size); + BufferType::set_dynamic_array_size(e1, _entry_size, 2); + new (static_cast<void *>(e1)) WrapInt32(42); + new (static_cast<void *>(e1 + 1)) WrapInt32(47); + new (static_cast<void *>(e1 + 2)) WrapInt32(49); // Not cleaned by clean_hold +} + +TEST_F(DynamicArrayBufferTypeTest, entry_size_is_calculated) +{ + EXPECT_EQ(8, get_entry_size<char>(1)); + EXPECT_EQ(8, get_entry_size<char>(2)); + EXPECT_EQ(8, get_entry_size<char>(3)); + EXPECT_EQ(8, get_entry_size<char>(4)); + EXPECT_EQ(12, get_entry_size<char>(5)); + EXPECT_EQ(8, get_entry_size<int16_t>(1)); + EXPECT_EQ(8, get_entry_size<int16_t>(2)); + EXPECT_EQ(12, get_entry_size<int16_t>(3)); + EXPECT_EQ(8, get_entry_size<int32_t>(1)); + EXPECT_EQ(12, get_entry_size<int32_t>(2)); + EXPECT_EQ(16, get_entry_size<int64_t>(1)); + EXPECT_EQ(24, get_entry_size<int64_t>(2)); + EXPECT_EQ(20, get_entry_size<WrapInt32>(4)); +} + +TEST_F(DynamicArrayBufferTypeTest, initialize_reserved_entries) +{ + _buffer_type.initialize_reserved_entries(_buf.get(), 2); + EXPECT_EQ((std::vector<int>{}), get_vector(_buf.get(), 0)); + EXPECT_EQ((std::vector<int>{}), get_vector(_buf.get(), 1)); + EXPECT_EQ((std::vector<int>{0, 0, 0}), get_max_vector(_buf.get(), 0)); + EXPECT_EQ((std::vector<int>{0, 0, 0}), get_max_vector(_buf.get(), 1)); + EXPECT_EQ(Counts(0, 0, 6, 0, 0), counts); +} + +TEST_F(DynamicArrayBufferTypeTest, fallback_copy) +{ + _buffer_type.initialize_reserved_entries(_buf.get(), 1); + write_entry1(); + EXPECT_EQ(Counts(0, 3, 3, 0, 0), counts); + auto buf2 = std::make_unique<char[]>(_buf_size); + _buffer_type.fallback_copy(buf2.get(), _buf.get(), 2); + EXPECT_EQ((std::vector<int>{}), get_vector(buf2.get(), 0)); + EXPECT_EQ((std::vector<int>{42, 47}), get_vector(buf2.get(), 1)); + EXPECT_EQ((std::vector<int>{0, 0, 0}), get_max_vector(buf2.get(), 0)); + EXPECT_EQ((std::vector<int>{42, 47, 49}), get_max_vector(buf2.get(), 1)); + EXPECT_EQ(Counts(0, 3, 9, 0, 0), counts); +} + +TEST_F(DynamicArrayBufferTypeTest, destroy_entries) +{ + _buffer_type.initialize_reserved_entries(_buf.get(), 2); + write_entry1(); + _buffer_type.destroy_entries(_buf.get(), 2); + EXPECT_EQ(Counts(0, 3, 6, 6, 0), counts); +} + +TEST_F(DynamicArrayBufferTypeTest, clean_hold) +{ + _buffer_type.initialize_reserved_entries(_buf.get(), 1); + write_entry1(); + MyCleanContext clean_context; + _buffer_type.clean_hold(_buf.get(), 1, 1, clean_context); + EXPECT_EQ((std::vector<int>{0, 0}), get_vector(_buf.get(), 1)); + EXPECT_EQ((std::vector<int>{0, 0, 49}), get_max_vector(_buf.get(), 1)); + EXPECT_EQ(Counts(0, 3, 3, 0, 2), counts); + _buffer_type.clean_hold(_buf.get(), 0, 2, clean_context); + EXPECT_EQ(Counts(0, 3, 3, 0, 4), counts); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt index 1c3b9112dda..38e5cb40abe 100644 --- a/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/datastore/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(vespalib_vespalib_datastore OBJECT SOURCES array_store.cpp array_store_config.cpp + array_store_dynamic_type_mapper.cpp array_store_type_mapper.cpp atomic_entry_ref.cpp buffer_free_list.cpp @@ -16,6 +17,7 @@ vespa_add_library(vespalib_vespalib_datastore OBJECT compaction_strategy.cpp datastore.cpp datastorebase.cpp + dynamic_array_buffer_type.cpp entry_ref_filter.cpp entryref.cpp fixed_size_hash_map.cpp diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.cpp b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.cpp new file mode 100644 index 00000000000..106a1f037d7 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.cpp @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "array_store_dynamic_type_mapper.hpp" + +namespace vespalib::datastore { + +template class ArrayStoreDynamicTypeMapper<char>; +template class ArrayStoreDynamicTypeMapper<int8_t>; +template class ArrayStoreDynamicTypeMapper<int16_t>; +template class ArrayStoreDynamicTypeMapper<int32_t>; +template class ArrayStoreDynamicTypeMapper<int64_t>; +template class ArrayStoreDynamicTypeMapper<float>; +template class ArrayStoreDynamicTypeMapper<double>; +template class ArrayStoreDynamicTypeMapper<AtomicEntryRef>; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.h b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.h new file mode 100644 index 00000000000..e02f57eaea3 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.h @@ -0,0 +1,53 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "array_store_type_mapper.h" +#include "atomic_entry_ref.h" + +namespace vespalib::datastore { + +template <typename EntryT> class SmallArrayBufferType; +template <typename EntryT> class DynamicArrayBufferType; +template <typename EntryT> class LargeArrayBufferType; + +/* + * This class provides mapping between type ids and array sizes needed for + * storing values. + * + * Type ids [1;max_static_array_buffer_type_id] use SmallBufferType, + * containing small arrays where buffer type specifies array size. + * + * Type ids [max_static_array_buffer_type_id+1;max_buffer_type_id] use + * DynamicBufferType, containing medium sized arrays where the same + * buffer type handles a range of array sizes and actual array size is + * also stored in the entry. + * + * Type id 0 uses LargeBufferType, which handles any array size but uses + * heap allocation. + */ +template <typename ElemT> +class ArrayStoreDynamicTypeMapper : public vespalib::datastore::ArrayStoreTypeMapper +{ + uint32_t _max_static_array_buffer_type_id; +public: + using SmallBufferType = vespalib::datastore::SmallArrayBufferType<ElemT>; + using DynamicBufferType = vespalib::datastore::DynamicArrayBufferType<ElemT>; + using LargeBufferType = vespalib::datastore::LargeArrayBufferType<ElemT>; + + ArrayStoreDynamicTypeMapper(uint32_t max_buffer_type_id, double grow_factor); + ~ArrayStoreDynamicTypeMapper(); + void setup_array_sizes(uint32_t max_buffer_type_id, double grow_factor); + size_t get_entry_size(uint32_t type_id) const; +}; + +extern template class ArrayStoreDynamicTypeMapper<char>; +extern template class ArrayStoreDynamicTypeMapper<int8_t>; +extern template class ArrayStoreDynamicTypeMapper<int16_t>; +extern template class ArrayStoreDynamicTypeMapper<int32_t>; +extern template class ArrayStoreDynamicTypeMapper<int64_t>; +extern template class ArrayStoreDynamicTypeMapper<float>; +extern template class ArrayStoreDynamicTypeMapper<double>; +extern template class ArrayStoreDynamicTypeMapper<AtomicEntryRef>; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp new file mode 100644 index 00000000000..f529ecccb46 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/array_store_dynamic_type_mapper.hpp @@ -0,0 +1,70 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "array_store_dynamic_type_mapper.h" +#include "dynamic_array_buffer_type.h" +#include "aligner.h" +#include <algorithm> +#include <cmath> +#include <limits> +#include <optional> + +namespace vespalib::datastore { + +template <typename ElemT> +ArrayStoreDynamicTypeMapper<ElemT>::ArrayStoreDynamicTypeMapper(uint32_t max_buffer_type_id, double grow_factor) + : ArrayStoreTypeMapper(), + _max_static_array_buffer_type_id(0) +{ + setup_array_sizes(max_buffer_type_id, grow_factor); +} + +template <typename ElemT> +void +ArrayStoreDynamicTypeMapper<ElemT>::setup_array_sizes(uint32_t max_buffer_type_id, double grow_factor) +{ + _array_sizes.clear(); + _array_sizes.reserve(max_buffer_type_id + 1); + _array_sizes.emplace_back(0); // type id 0 is fallback for large arrays + size_t array_size = 1u; + size_t entry_size = sizeof(ElemT); + bool dynamic_arrays = false; + for (uint32_t type_id = 1; type_id <= max_buffer_type_id; ++type_id) { + if (type_id > 1) { + array_size = std::max(array_size + 1, static_cast<size_t>(std::floor(array_size * grow_factor))); + if (array_size > _array_sizes.back() + 1 || dynamic_arrays) { + if (!dynamic_arrays) { + _max_static_array_buffer_type_id = type_id - 1; + dynamic_arrays = true; + } + entry_size = DynamicBufferType::calc_entry_size(array_size); + array_size = DynamicBufferType::calc_array_size(entry_size); + } else { + entry_size = array_size * sizeof(ElemT); + } + } + if (entry_size > std::numeric_limits<uint32_t>::max()) { + break; + } + _array_sizes.emplace_back(array_size); + } + if (!dynamic_arrays) { + _max_static_array_buffer_type_id = _array_sizes.size() - 1; + } +} + +template <typename ElemT> +ArrayStoreDynamicTypeMapper<ElemT>::~ArrayStoreDynamicTypeMapper() = default; + +template <typename ElemT> +size_t +ArrayStoreDynamicTypeMapper<ElemT>::get_entry_size(uint32_t type_id) const +{ + auto array_size = get_array_size(type_id); + if (type_id <= _max_static_array_buffer_type_id) { + return array_size * sizeof(ElemT); + } else { + return DynamicBufferType::calc_entry_size(array_size); + } +} + +} diff --git a/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.cpp new file mode 100644 index 00000000000..df2129e8ff2 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.cpp @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dynamic_array_buffer_type.hpp" + +namespace vespalib::datastore { + +template class DynamicArrayBufferType<char>; +template class DynamicArrayBufferType<int8_t>; +template class DynamicArrayBufferType<int16_t>; +template class DynamicArrayBufferType<int32_t>; +template class DynamicArrayBufferType<int64_t>; +template class DynamicArrayBufferType<float>; +template class DynamicArrayBufferType<double>; +template class DynamicArrayBufferType<AtomicEntryRef>; + +} + diff --git a/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.h new file mode 100644 index 00000000000..e314accd664 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.h @@ -0,0 +1,58 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "buffer_type.h" + +namespace vespalib::datastore { + +/** + * Concrete class used to manage allocation and de-allocation of + * elements of type ElemType in data store buffers. + * + * Layout of each entry is: + * + * elements[array_size] - array of elements in entry + * padding - to align entries + * dynamic_array_size - number of array elements that should + * be visible to reader. + */ +template <typename ElemT> +class DynamicArrayBufferType : public BufferTypeBase +{ +public: + using ElemType = ElemT; +protected: + static const ElemType& empty_entry() noexcept; + ElemType* get_entry(void *buffer, size_t offset) noexcept { return get_entry(buffer, offset, entry_size()); } + const ElemType* get_entry(const void *buffer, size_t offset) const noexcept { return get_entry(buffer, offset, entry_size()); } +public: + DynamicArrayBufferType(const DynamicArrayBufferType &rhs) = delete; + DynamicArrayBufferType & operator=(const DynamicArrayBufferType &rhs) = delete; + DynamicArrayBufferType(DynamicArrayBufferType && rhs) noexcept = default; + DynamicArrayBufferType & operator=(DynamicArrayBufferType && rhs) noexcept = default; + DynamicArrayBufferType(uint32_t array_size, uint32_t min_entries, uint32_t max_entries, + uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept; + ~DynamicArrayBufferType() override; + void destroy_entries(void* buffer, EntryCount num_entries) override; + void fallback_copy(void* new_buffer, const void* old_buffer, EntryCount num_entries) override; + void initialize_reserved_entries(void* buffer, EntryCount reserved_entries) override; + void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCxt) override; + static size_t calc_entry_size(size_t array_size) noexcept; + static size_t calc_array_size(size_t entry_size) noexcept; + static ElemType* get_entry(void* buffer, size_t offset, uint32_t entry_size) noexcept { return reinterpret_cast<ElemType*>(static_cast<char*>(buffer) + offset * entry_size); } + static const ElemType* get_entry(const void* buffer, size_t offset, uint32_t entry_size) noexcept { return reinterpret_cast<const ElemType*>(static_cast<const char*>(buffer) + offset * entry_size); } + static uint32_t get_dynamic_array_size(const void *buffer, uint32_t entry_size) noexcept { return *reinterpret_cast<const uint32_t*>(static_cast<const char*>(buffer) + entry_size - sizeof(uint32_t)); } + static void set_dynamic_array_size(void *buffer, uint32_t entry_size, uint32_t array_size) noexcept { *reinterpret_cast<uint32_t*>(static_cast<char*>(buffer) + entry_size - sizeof(uint32_t)) = array_size; } +}; + +extern template class DynamicArrayBufferType<char>; +extern template class DynamicArrayBufferType<int8_t>; +extern template class DynamicArrayBufferType<int16_t>; +extern template class DynamicArrayBufferType<int32_t>; +extern template class DynamicArrayBufferType<int64_t>; +extern template class DynamicArrayBufferType<float>; +extern template class DynamicArrayBufferType<double>; +extern template class DynamicArrayBufferType<AtomicEntryRef>; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.hpp new file mode 100644 index 00000000000..514267b0a85 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/dynamic_array_buffer_type.hpp @@ -0,0 +1,111 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "dynamic_array_buffer_type.h" +#include "aligner.h" +#include <algorithm> +#include <cassert> + +namespace vespalib::datastore { + +template <typename ElemT> +DynamicArrayBufferType<ElemT>::DynamicArrayBufferType(uint32_t array_size, uint32_t min_entries, uint32_t max_entries, + uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept + : BufferTypeBase(calc_entry_size(array_size), array_size, min_entries, max_entries, num_entries_for_new_buffer, allocGrowFactor) +{ } + +template <typename ElemT> +DynamicArrayBufferType<ElemT>::~DynamicArrayBufferType() = default; + +template <typename ElemT> +size_t +DynamicArrayBufferType<ElemT>::calc_entry_size(size_t array_size) noexcept +{ + Aligner aligner(std::max(alignof(uint32_t), alignof(ElemType))); + return aligner.align(sizeof(ElemType) * array_size + sizeof(uint32_t)); +} + +template <typename ElemT> +size_t +DynamicArrayBufferType<ElemT>::calc_array_size(size_t entry_size) noexcept +{ + return (entry_size - sizeof(uint32_t)) / sizeof(ElemType); +} + +template <typename ElemT> +void +DynamicArrayBufferType<ElemT>::destroy_entries(void* buffer, EntryCount num_entries) +{ + uint32_t array_size = _arraySize; + for (uint32_t entry_idx = 0; entry_idx < num_entries; ++entry_idx) { + auto e = get_entry(buffer, entry_idx); + for (uint32_t elem_idx = 0; elem_idx < array_size; ++elem_idx) { + e->~ElemType(); + ++e; + } + } +} + +template <typename ElemT> +void +DynamicArrayBufferType<ElemT>::fallback_copy(void* new_buffer, const void* old_buffer, EntryCount num_entries) +{ + uint32_t array_size = _arraySize; + for (uint32_t entry_idx = 0; entry_idx < num_entries; ++entry_idx) { + auto d = get_entry(new_buffer, entry_idx); + auto s = get_entry(old_buffer, entry_idx); + set_dynamic_array_size(d, entry_size(), get_dynamic_array_size(s, entry_size())); + for (uint32_t elem_idx = 0; elem_idx < array_size; ++elem_idx) { + new (static_cast<void*>(d)) ElemType(*s); + ++s; + ++d; + } + } +} + +template <typename ElemT> +void +DynamicArrayBufferType<ElemT>::initialize_reserved_entries(void* buffer, EntryCount reserved_entries) +{ + uint32_t array_size = _arraySize; + const auto& empty = empty_entry(); + for (uint32_t entry_idx = 0; entry_idx < reserved_entries; ++entry_idx) { + auto e = get_entry(buffer, entry_idx); + set_dynamic_array_size(e, entry_size(), 0); + for (uint32_t elem_idx = 0; elem_idx < array_size; ++elem_idx) { + new (static_cast<void*>(e)) ElemType(empty); + ++e; + } + } +} + +template <typename ElemT> +void +DynamicArrayBufferType<ElemT>::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext) +{ + uint32_t max_array_size = _arraySize; + const auto& empty = empty_entry(); + for (uint32_t entry_idx = 0; entry_idx < num_entries; ++entry_idx) { + auto e = get_entry(buffer, offset + entry_idx); + auto array_size = get_dynamic_array_size(e, entry_size()); + assert(array_size <= max_array_size); + for (uint32_t elem_idx = 0; elem_idx < array_size; ++elem_idx) { + *e = empty; + ++e; + } + } +} + +template <typename ElemT> +const ElemT& +DynamicArrayBufferType<ElemT>::empty_entry() noexcept +{ + // It's possible for ElemType to wrap e.g. an Alloc instance, which has a transitive + // dependency on globally constructed allocator object(s). To avoid issues with global + // construction order, initialize the sentinel on the first access. + static ElemType empty = ElemType(); + return empty; +} + +} |