diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-10-26 16:40:36 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2020-10-26 16:40:36 +0000 |
commit | 9b7cc9c30b8333bdd2d06696ff27ce97fb130c9d (patch) | |
tree | a0d39ecdbb22c138abd6648eca3da382fbbb4f2f /storageapi/src | |
parent | 22f126bc61fd6bb16871132f95ec5073f69cedff (diff) |
- Honour max merge chunk size in one place.
- GC incomplete enable_merge_local_node_choose_docs_optimalization handling.
Diffstat (limited to 'storageapi/src')
5 files changed, 11 insertions, 21 deletions
diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index d1de3bf5afb..9fd2c96f27e 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -688,7 +688,7 @@ TEST_P(StorageProtocolTest, apply_bucket_diff) { nodes.push_back(13); std::vector<ApplyBucketDiffCommand::Entry> entries = {dummy_apply_entry()}; - auto cmd = std::make_shared<ApplyBucketDiffCommand>(_bucket, nodes, 1234); + auto cmd = std::make_shared<ApplyBucketDiffCommand>(_bucket, nodes); cmd->getDiff() = entries; auto cmd2 = copyCommand(cmd); EXPECT_EQ(_bucket, cmd2->getBucket()); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index 13fba8b8508..c57b991eb36 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -193,7 +193,7 @@ void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::ApplyBucketDiffCo buf.putShort(nodes[i].index); buf.putBoolean(nodes[i].sourceOnly); } - buf.putInt(msg.getMaxBufferSize()); + buf.putInt(0x400000); const std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg.getDiff()); buf.putInt(entries.size()); for (uint32_t i=0; i<entries.size(); ++i) { @@ -220,12 +220,12 @@ ProtocolSerialization4_2::onDecodeApplyBucketDiffCommand(BBuf& buf) const bool sourceOnly = SH::getBoolean(buf); nodes.push_back(Node(index, sourceOnly)); } - uint32_t maxBufferSize(SH::getInt(buf)); - auto msg = std::make_unique<api::ApplyBucketDiffCommand>(bucket, nodes, maxBufferSize); + (void) SH::getInt(buf); // Unused field + auto msg = std::make_unique<api::ApplyBucketDiffCommand>(bucket, nodes); std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg->getDiff()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { - // Trigger out of bounds exception rather than out of memory error + // Trigger out of bounds exception rather than out of memory error buf.incPos(entryCount); } entries.resize(entryCount); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 166925382bd..5d66d86036c 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -944,7 +944,7 @@ void fill_proto_apply_diff_vector(::google::protobuf::RepeatedPtrField<protobuf: void ProtocolSerialization7::onEncode(GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const { encode_bucket_request<protobuf::ApplyBucketDiffRequest>(buf, msg, [&](auto& req) { set_merge_nodes(*req.mutable_nodes(), msg.getNodes()); - req.set_max_buffer_size(msg.getMaxBufferSize()); + req.set_max_buffer_size(0x400000); // Unused, GC soon. fill_proto_apply_diff_vector(*req.mutable_entries(), msg.getDiff()); }); } @@ -958,7 +958,7 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::ApplyBucketDiffRepl api::StorageCommand::UP ProtocolSerialization7::onDecodeApplyBucketDiffCommand(BBuf& buf) const { return decode_bucket_request<protobuf::ApplyBucketDiffRequest>(buf, [&](auto& req, auto& bucket) { auto nodes = get_merge_nodes(req.nodes()); - auto cmd = std::make_unique<api::ApplyBucketDiffCommand>(bucket, std::move(nodes), req.max_buffer_size()); + auto cmd = std::make_unique<api::ApplyBucketDiffCommand>(bucket, std::move(nodes)); fill_api_apply_diff_vector(cmd->getDiff(), req.entries()); return cmd; }); diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp index ced71f5326b..1262c5db632 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.cpp +++ b/storageapi/src/vespa/storageapi/message/bucket.cpp @@ -341,12 +341,10 @@ ApplyBucketDiffCommand::Entry::operator==(const Entry& e) const } ApplyBucketDiffCommand::ApplyBucketDiffCommand( - const document::Bucket &bucket, const std::vector<Node>& nodes, - uint32_t maxBufferSize) + const document::Bucket &bucket, const std::vector<Node>& nodes) : BucketInfoCommand(MessageType::APPLYBUCKETDIFF, bucket), _nodes(nodes), - _diff(), - _maxBufferSize(maxBufferSize) + _diff() {} ApplyBucketDiffCommand::~ApplyBucketDiffCommand() = default; @@ -369,8 +367,7 @@ ApplyBucketDiffCommand::print(std::ostream& out, bool verbose, if (i != 0) out << ", "; out << _nodes[i]; } - out << ", max buffer size " << _maxBufferSize << " bytes" - << ", " << _diff.size() << " entries of " << totalSize << " bytes, " + out << _diff.size() << " entries of " << totalSize << " bytes, " << (100.0 * filled / _diff.size()) << " \% filled)"; if (_diff.empty()) { out << ", no entries"; diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h index eabe847c7e0..e5ec7698329 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.h +++ b/storageapi/src/vespa/storageapi/message/bucket.h @@ -274,22 +274,15 @@ public: private: std::vector<Node> _nodes; std::vector<Entry> _diff; - // We may send more metadata entries that should fit in one apply bucket - // diff command, to let node pick which ones it wants to fill in first. - // Nodes should verify that they don't fill a command up with more than - // this number of bytes. - uint32_t _maxBufferSize; public: ApplyBucketDiffCommand(const document::Bucket &bucket, - const std::vector<Node>& nodes, - uint32_t maxBufferSize); + const std::vector<Node>& nodes); ~ApplyBucketDiffCommand() override; const std::vector<Node>& getNodes() const { return _nodes; } const std::vector<Entry>& getDiff() const { return _diff; } std::vector<Entry>& getDiff() { return _diff; } - uint32_t getMaxBufferSize() const { return _maxBufferSize; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; DECLARE_STORAGECOMMAND(ApplyBucketDiffCommand, onApplyBucketDiff) |