From 86bd7f85ee0f24dfb62a4c20da464d1bcb390bfa Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 16 Mar 2020 10:56:16 +0000 Subject: Bump max number of groups from 16M to 4G --- searchlib/src/tests/grouping/grouping_test.cpp | 1 + searchlib/src/vespa/searchlib/aggregation/group.cpp | 20 ++++++++++++-------- searchlib/src/vespa/searchlib/aggregation/group.h | 14 ++++++-------- 3 files changed, 19 insertions(+), 16 deletions(-) (limited to 'searchlib') diff --git a/searchlib/src/tests/grouping/grouping_test.cpp b/searchlib/src/tests/grouping/grouping_test.cpp index 94d3afd619b..d1430322c26 100644 --- a/searchlib/src/tests/grouping/grouping_test.cpp +++ b/searchlib/src/tests/grouping/grouping_test.cpp @@ -307,6 +307,7 @@ void Test::testAggregationSimple() { EXPECT_EQUAL(64u, sizeof(Group)); + EXPECT_EQUAL(40u, sizeof(Group::Value)); AggregationContext ctx; ctx.result().add(0).add(1).add(2); ctx.add(IntAttrBuilder("int").add(3).add(7).add(15).sp()); diff --git a/searchlib/src/vespa/searchlib/aggregation/group.cpp b/searchlib/src/vespa/searchlib/aggregation/group.cpp index 1808af79b4d..d6d3a8f1deb 100644 --- a/searchlib/src/vespa/searchlib/aggregation/group.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/group.cpp @@ -291,7 +291,6 @@ void Group::Value::addChild(Group * child) { const size_t sz(getChildrenSize()); - assert(sz < 0xffffff); if (_children == nullptr) { _children = new ChildP[4]; } else if ((sz >=4) && vespalib::Optimized::msbIdx(sz) == vespalib::Optimized::lsbIdx(sz)) { @@ -631,11 +630,12 @@ Group::Value::visitMembers(vespalib::ObjectVisitor &visitor) const { } Group::Value::Value() : - _packedLength(0), - _tag(-1), _aggregationResults(nullptr), _children(nullptr), _childInfo(), + _childrenLength(0), + _tag(-1), + _packedLength(0), _orderBy() { memset(_orderBy, 0, sizeof(_orderBy)); @@ -643,11 +643,12 @@ Group::Value::Value() : } Group::Value::Value(const Value & rhs) : - _packedLength(rhs._packedLength), - _tag(rhs._tag), _aggregationResults(nullptr), _children(nullptr), _childInfo(), + _childrenLength(rhs._childrenLength), + _tag(rhs._tag), + _packedLength(rhs._packedLength), _orderBy() { _childInfo._childMap = nullptr; @@ -671,11 +672,12 @@ Group::Value::Value(const Value & rhs) : } Group::Value::Value(Value && rhs) noexcept : - _packedLength(rhs._packedLength), - _tag(rhs._tag), _aggregationResults(rhs._aggregationResults), _children(rhs._children), _childInfo(rhs._childInfo), + _childrenLength(rhs._childrenLength), + _tag(rhs._tag), + _packedLength(rhs._packedLength), _orderBy() { memcpy(_orderBy, rhs._orderBy, sizeof(_orderBy)); @@ -688,8 +690,9 @@ Group::Value::Value(Value && rhs) noexcept : Group::Value & Group::Value::operator =(Value && rhs) noexcept { - _packedLength = rhs._packedLength; + _childrenLength = rhs._childrenLength; _tag = rhs._tag; + _packedLength = rhs._packedLength; _aggregationResults = rhs._aggregationResults; _children = rhs._children; _childInfo = rhs._childInfo; @@ -729,6 +732,7 @@ Group::Value::swap(Value & rhs) memcpy(_orderBy, rhs._orderBy, sizeof(_orderBy)); memcpy(rhs._orderBy, tmp, sizeof(_orderBy)); } + std::swap(_childrenLength, rhs._childrenLength); std::swap(_tag, rhs._tag); std::swap(_packedLength, rhs._packedLength); } diff --git a/searchlib/src/vespa/searchlib/aggregation/group.h b/searchlib/src/vespa/searchlib/aggregation/group.h index f302346f211..2c0e72b543b 100644 --- a/searchlib/src/vespa/searchlib/aggregation/group.h +++ b/searchlib/src/vespa/searchlib/aggregation/group.h @@ -58,8 +58,6 @@ public: using GroupingLevelList = std::vector; -private: - class Value { public: Value(); @@ -101,7 +99,7 @@ private: void addChild(Group * child); uint32_t getAggrSize() const { return _packedLength & 0x0f; } uint32_t getOrderBySize() const { return (_packedLength >> 6) & 0x03; } - uint32_t getChildrenSize() const { return (_packedLength >> 8); } + uint32_t getChildrenSize() const { return _childrenLength; } uint32_t getExpr(uint32_t i) const { return getAggrSize() + i; } int32_t getOrderBy(uint32_t i) const { int32_t v((_orderBy[i/2] >> (4*(i%2))) & 0x0f); @@ -122,7 +120,7 @@ private: void setAggrSize(uint32_t v) { _packedLength = (_packedLength & ~0x0f) | v; } void setExprSize(uint32_t v) { _packedLength = (_packedLength & ~0x30) | (v << 4); } void setOrderBySize(uint32_t v) { _packedLength = (_packedLength & ~0xc0) | (v << 6); } - void setChildrenSize(uint32_t v) { _packedLength = (_packedLength & ~0xffffff00) | (v << 8); } + void setChildrenSize(uint32_t v) { _childrenLength = v; } AggregationResult * getAggr(size_t i) { return static_cast(_aggregationResults[i].get()); } const AggregationResult & getAggr(size_t i) const { return static_cast(*_aggregationResults[i]); } const ExpressionNode::CP & getAggrCP(size_t i) const { return _aggregationResults[i]; } @@ -139,9 +137,6 @@ private: } bool needFullRank() const { return getOrderBySize() != 0; } - uint32_t _packedLength; // Length of the 3 vectors below - uint32_t _tag; // Opaque tag used to identify the group by the client. - // The collectors and expressions stored by this group. Currently, both aggregation results and expressions used by orderby() are stored in this // array to save 8 bytes in the Group size. This makes it important to use the getAggr() and expr() methods for accessing elements, // as they will correctly offset the index to the correct place in the array. @@ -152,9 +147,12 @@ private: GroupHash *_childMap; // child map used during aggregation size_t _allChildren; // Keep real number of children. } _childInfo; + uint32_t _childrenLength; + uint32_t _tag; // Opaque tag used to identify the group by the client. + uint8_t _packedLength; // Length of aggr and expr vectors. uint8_t _orderBy[2]; // How this group is ranked, negative means reverse rank. }; - +private: ResultNode::CP _id; // the label of this group, separating it from other groups RawRank _rank; // The default rank taken from the highest hit relevance. Value _aggr; -- cgit v1.2.3