diff options
31 files changed, 129 insertions, 61 deletions
diff --git a/document/src/vespa/document/test/CMakeLists.txt b/document/src/vespa/document/test/CMakeLists.txt index f16bfe0144a..1620a8fad39 100644 --- a/document/src/vespa/document/test/CMakeLists.txt +++ b/document/src/vespa/document/test/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(document_test OBJECT SOURCES + make_bucket_space.cpp make_document_bucket.cpp DEPENDS ) diff --git a/document/src/vespa/document/test/make_bucket_space.cpp b/document/src/vespa/document/test/make_bucket_space.cpp new file mode 100644 index 00000000000..a213e5e36b7 --- /dev/null +++ b/document/src/vespa/document/test/make_bucket_space.cpp @@ -0,0 +1,12 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "make_bucket_space.h" + +namespace document::test { + +BucketSpace makeBucketSpace() +{ + return BucketSpace::placeHolder(); +} + +} diff --git a/document/src/vespa/document/test/make_bucket_space.h b/document/src/vespa/document/test/make_bucket_space.h new file mode 100644 index 00000000000..8289ec2030a --- /dev/null +++ b/document/src/vespa/document/test/make_bucket_space.h @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/document/bucket/bucketspace.h> + +namespace document::test { + +// Helper function used by unit tests + +BucketSpace makeBucketSpace(); + +} diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp index f25e3f19b08..d4052f30958 100644 --- a/documentapi/src/tests/messages/messages50test.cpp +++ b/documentapi/src/tests/messages/messages50test.cpp @@ -12,7 +12,6 @@ using document::DataType; using document::DocumentTypeRepo; using document::test::makeDocumentBucket; -using document::BucketSpace; /////////////////////////////////////////////////////////////////////////////// // @@ -1113,7 +1112,7 @@ Messages50Test::testWrongDistributionReply() bool Messages50Test::testGetBucketListReply() { - GetBucketListReply reply(BucketSpace::placeHolder()); + GetBucketListReply reply; reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(16, 123), "foo")); reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(17, 1123), "bar")); reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(18, 11123), "zoink")); diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp index 974135972f6..49e8e048db5 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp @@ -16,7 +16,7 @@ GetBucketListMessage::GetBucketListMessage(const document::Bucket &bucket) : DocumentReply::UP GetBucketListMessage::doCreateReply() const { - return DocumentReply::UP(new GetBucketListReply(_bucket.getBucketSpace())); + return DocumentReply::UP(new GetBucketListReply()); } uint32_t diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp index e9f66ed1f2f..e144158e0ef 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp @@ -23,9 +23,8 @@ GetBucketListReply::BucketInfo::operator==(const GetBucketListReply::BucketInfo return _bucket == rhs._bucket && _bucketInformation == rhs._bucketInformation; } -GetBucketListReply::GetBucketListReply(document::BucketSpace bucketSpace) : +GetBucketListReply::GetBucketListReply() : DocumentReply(DocumentProtocol::REPLY_GETBUCKETLIST), - _bucketSpace(bucketSpace), _buckets() { // empty diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.h index 3c6a2a25bfc..3ded7f7d08b 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.h @@ -2,7 +2,7 @@ #pragma once #include "documentreply.h" -#include <vespa/document/bucket/bucket.h> +#include <vespa/document/bucket/bucketid.h> namespace documentapi { @@ -20,14 +20,13 @@ public: }; private: - document::BucketSpace _bucketSpace; std::vector<BucketInfo> _buckets; public: /** * Constructs a new reply with no content. */ - GetBucketListReply(document::BucketSpace bucketSpace); + GetBucketListReply(); /** * Returns the bucket state contained in this. diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp index fff46c0128d..6876b4c3a71 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp @@ -424,7 +424,7 @@ RoutableFactories50::GetBucketListMessageFactory::doEncode(const DocumentMessage DocumentReply::UP RoutableFactories50::GetBucketListReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new GetBucketListReply(BucketSpace::placeHolder())); + DocumentReply::UP ret(new GetBucketListReply()); GetBucketListReply &reply = static_cast<GetBucketListReply&>(*ret); int32_t len = decodeInt(buf); diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index 98f3e06fc62..8cc046861be 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -15,6 +15,7 @@ #include <tests/common/dummystoragelink.h> #include <tests/common/testhelper.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/vdslib/state/random.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/testkit/testapp.h> @@ -31,6 +32,7 @@ using config::FileSpec; using document::DocumentType; using document::DocumentTypeRepo; using document::test::makeDocumentBucket; +using document::test::makeBucketSpace; namespace storage { @@ -415,13 +417,13 @@ void BucketManagerTest::testRequestBucketInfoWithState() // Send a request bucket info command that will be outdated and failed. std::shared_ptr<api::RequestBucketInfoCommand> cmd1( - new api::RequestBucketInfoCommand(0, states[1])); + new api::RequestBucketInfoCommand(makeBucketSpace(), 0, states[1])); // Send two request bucket info commands that will be processed together // when the bucket manager is idle, as states are equivalent std::shared_ptr<api::RequestBucketInfoCommand> cmd2( - new api::RequestBucketInfoCommand(0, states[2])); + new api::RequestBucketInfoCommand(makeBucketSpace(), 0, states[2])); std::shared_ptr<api::RequestBucketInfoCommand> cmd3( - new api::RequestBucketInfoCommand(0, states[3])); + new api::RequestBucketInfoCommand(makeBucketSpace(), 0, states[3])); // Tag server initialized before starting _top->open(); @@ -559,7 +561,7 @@ void BucketManagerTest::testRequestBucketInfoWithList() bids.push_back(document::BucketId(16, 0xe8c8)); std::shared_ptr<api::RequestBucketInfoCommand> cmd( - new api::RequestBucketInfoCommand(bids)); + new api::RequestBucketInfoCommand(makeBucketSpace(), bids)); _top->sendDown(cmd); _top->waitForMessages(1, 5); @@ -729,11 +731,11 @@ public: } auto createFullFetchCommand() const { - return std::make_shared<api::RequestBucketInfoCommand>(0, _state); + return std::make_shared<api::RequestBucketInfoCommand>(makeBucketSpace(), 0, _state); } auto createFullFetchCommandWithHash(vespalib::stringref hash) const { - return std::make_shared<api::RequestBucketInfoCommand>(0, _state, hash); + return std::make_shared<api::RequestBucketInfoCommand>(makeBucketSpace(), 0, _state, hash); } auto acquireBucketLockAndSendInfoRequest(const document::BucketId& bucket) { @@ -907,7 +909,7 @@ BucketManagerTest::testOrderRepliesAfterBucketSpecificRequest() auto infoRoundtrip = std::async(std::launch::async, [&]() { std::vector<document::BucketId> buckets{bucketA}; - auto infoCmd = std::make_shared<api::RequestBucketInfoCommand>(buckets); + auto infoCmd = std::make_shared<api::RequestBucketInfoCommand>(makeBucketSpace(), buckets); // Can't complete until `guard` has been unlocked. _top->sendDown(infoCmd); // Barrier: bucket reply and subsequent split reply @@ -947,7 +949,7 @@ BucketManagerTest::testQueuedRepliesOnlyDispatchedWhenAllProcessingDone() auto singleBucketInfo = std::async(std::launch::async, [&]() { std::vector<document::BucketId> buckets{bucketA}; - auto infoCmd = std::make_shared<api::RequestBucketInfoCommand>(buckets); + auto infoCmd = std::make_shared<api::RequestBucketInfoCommand>(makeBucketSpace(), buckets); _top->sendDown(infoCmd); _top->waitForMessages(3, MESSAGE_WAIT_TIME); }); @@ -1209,7 +1211,7 @@ void BucketManagerTest::sendSingleBucketInfoRequest(const document::BucketId& id) { std::vector<document::BucketId> buckets{id}; - auto infoCmd = std::make_shared<api::RequestBucketInfoCommand>(buckets); + auto infoCmd = std::make_shared<api::RequestBucketInfoCommand>(makeBucketSpace(), buckets); _top->sendDown(infoCmd); } diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index b830aa6d506..4be9735c0f7 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -10,12 +10,14 @@ #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> #include <tests/distributor/distributortestutil.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/storage/config/config-stor-distributormanager.h> #include <tests/common/dummystoragelink.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/vespalib/text/stringtokenizer.h> using document::test::makeDocumentBucket; +using document::test::makeBucketSpace; namespace storage { @@ -209,7 +211,7 @@ Distributor_Test::testOperationGeneration() document::DocumentId("userdoc:m:1:foo"), api::Timestamp(1234)))); - api::CreateVisitorCommand* cmd = new api::CreateVisitorCommand("foo", "bar", ""); + api::CreateVisitorCommand* cmd = new api::CreateVisitorCommand(makeBucketSpace(), "foo", "bar", ""); cmd->addBucketToBeVisited(document::BucketId(16, 1)); cmd->addBucketToBeVisited(document::BucketId()); diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp index e48046fb4ac..d4d7a00a2df 100644 --- a/storage/src/tests/distributor/idealstatemanagertest.cpp +++ b/storage/src/tests/distributor/idealstatemanagertest.cpp @@ -10,8 +10,10 @@ #include <vespa/storageapi/message/bucketsplitting.h> #include <tests/distributor/distributortestutil.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/test/make_bucket_space.h> using document::test::makeDocumentBucket; +using document::test::makeBucketSpace; namespace storage { namespace distributor { @@ -198,7 +200,7 @@ IdealStateManagerTest::testBlockIdealStateOpsOnFullRequestBucketInfo() // sent to the entire node. It will then use a null bucketid. { std::shared_ptr<api::RequestBucketInfoCommand> msg( - new api::RequestBucketInfoCommand(buckets)); + new api::RequestBucketInfoCommand(makeBucketSpace(), buckets)); msg->setAddress( api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 4)); tracker.insert(msg); @@ -220,7 +222,7 @@ IdealStateManagerTest::testBlockIdealStateOpsOnFullRequestBucketInfo() // Don't block on null-bucket messages that aren't RequestBucketInfo. { std::shared_ptr<api::CreateVisitorCommand> msg( - new api::CreateVisitorCommand("foo", "bar", "baz")); + new api::CreateVisitorCommand(makeBucketSpace(), "foo", "bar", "baz")); msg->setAddress( api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 7)); tracker.insert(msg); diff --git a/storage/src/tests/distributor/visitoroperationtest.cpp b/storage/src/tests/distributor/visitoroperationtest.cpp index c8f339f6626..f12e1fa4e33 100644 --- a/storage/src/tests/distributor/visitoroperationtest.cpp +++ b/storage/src/tests/distributor/visitoroperationtest.cpp @@ -13,11 +13,13 @@ #include <vespa/storage/distributor/distributor.h> #include <tests/common/dummystoragelink.h> #include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/document/test/make_bucket_space.h> using namespace document; using namespace storage::api; using namespace storage::lib; using namespace std::string_literals; +using document::test::makeBucketSpace; namespace storage { namespace distributor { @@ -148,7 +150,7 @@ private: const std::string& docSelection = "") { api::CreateVisitorCommand::SP cmd( - new api::CreateVisitorCommand(libraryName, instanceId, docSelection)); + new api::CreateVisitorCommand(makeBucketSpace(), libraryName, instanceId, docSelection)); cmd->setControlDestination("controldestination"); cmd->setDataDestination("datadestination"); cmd->setFieldSet("[header]"); @@ -265,7 +267,8 @@ VisitorOperationTest::doStandardVisitTest(const std::string& clusterState) vespalib::string libraryName("dumpvisitor"); vespalib::string docSelection(""); api::CreateVisitorCommand::SP msg( - new api::CreateVisitorCommand(libraryName, + new api::CreateVisitorCommand(makeBucketSpace(), + libraryName, instanceId, docSelection)); vespalib::string controlDestination("controldestination"); @@ -333,7 +336,8 @@ VisitorOperationTest::testShutdown() vespalib::string libraryName("dumpvisitor"); vespalib::string docSelection(""); api::CreateVisitorCommand::SP msg( - new api::CreateVisitorCommand(libraryName, + new api::CreateVisitorCommand(makeBucketSpace(), + libraryName, instanceId, docSelection)); msg->addBucketToBeVisited(id); @@ -361,7 +365,7 @@ VisitorOperationTest::testNoBucket() // Send create visitor api::CreateVisitorCommand::SP msg(new api::CreateVisitorCommand( - "dumpvisitor", "instance", "")); + makeBucketSpace(), "dumpvisitor", "instance", "")); CPPUNIT_ASSERT_EQUAL(std::string( "CreateVisitorReply(last=BucketId(0x0000000000000000)) " @@ -377,7 +381,7 @@ VisitorOperationTest::testOnlySuperBucketAndProgressAllowed() // Send create visitor api::CreateVisitorCommand::SP msg(new api::CreateVisitorCommand( - "dumpvisitor", "instance", "")); + makeBucketSpace(), "dumpvisitor", "instance", "")); msg->addBucketToBeVisited(nullId); msg->addBucketToBeVisited(nullId); msg->addBucketToBeVisited(nullId); diff --git a/storage/src/tests/visiting/commandqueuetest.cpp b/storage/src/tests/visiting/commandqueuetest.cpp index 91f196a9339..7b6d5dcafd6 100644 --- a/storage/src/tests/visiting/commandqueuetest.cpp +++ b/storage/src/tests/visiting/commandqueuetest.cpp @@ -5,9 +5,11 @@ #include <vespa/storageapi/message/visitor.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/document/test/make_bucket_space.h> using vespalib::string; +using document::test::makeBucketSpace; namespace storage { @@ -39,7 +41,7 @@ namespace { ost << name << " t=" << timeout << " p=" << static_cast<unsigned int>(priority); // Piggyback name in document selection std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("", "", ost.str())); + new api::CreateVisitorCommand(makeBucketSpace(), "", "", ost.str())); cmd->setQueueTimeout(timeout); cmd->setPriority(priority); return cmd; diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index 3c5780cf77c..5da86a134e2 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -13,6 +13,7 @@ #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/test/make_bucket_space.h> #include <tests/storageserver/testvisitormessagesession.h> #include <vespa/documentapi/messagebus/messages/multioperationmessage.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> @@ -21,6 +22,7 @@ #include <vespa/vespalib/util/exceptions.h> using document::test::makeDocumentBucket; +using document::test::makeBucketSpace; namespace storage { namespace { @@ -410,7 +412,7 @@ VisitorManagerTest::testNormalUsage() initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setControlDestination("foo/bar"); @@ -436,7 +438,7 @@ VisitorManagerTest::testResending() initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setControlDestination("foo/bar"); @@ -486,7 +488,7 @@ VisitorManagerTest::testVisitEmptyBucket() addSomeRemoves(true); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); @@ -502,7 +504,7 @@ VisitorManagerTest::testMultiBucketVisit() initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); for (uint32_t i=0; i<10; ++i) { cmd->addBucketToBeVisited(document::BucketId(16, i)); } @@ -527,7 +529,7 @@ VisitorManagerTest::testNoBuckets() initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); cmd->setAddress(address); _top->sendDown(cmd); @@ -553,7 +555,7 @@ void VisitorManagerTest::testVisitPutsAndRemoves() addSomeRemoves(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); cmd->setAddress(address); cmd->setVisitRemoves(); for (uint32_t i=0; i<10; ++i) { @@ -581,7 +583,7 @@ void VisitorManagerTest::testVisitWithTimeframeAndSelection() initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", "testvis", + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "testdoctype1.headerval < 2")); cmd->setFromTime(3); cmd->setToTime(8); @@ -613,7 +615,7 @@ void VisitorManagerTest::testVisitWithTimeframeAndBogusSelection() initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", "testvis", + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "DocType(testdoctype1---///---) XXX BAD Field(headerval) < 2")); cmd->setFromTime(3); cmd->setToTime(8); @@ -641,7 +643,7 @@ VisitorManagerTest::testVisitorCallbacks() std::ostringstream replydata; api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("TestVisitor", "testvis", "")); + new api::CreateVisitorCommand(makeBucketSpace(), "TestVisitor", "testvis", "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->addBucketToBeVisited(document::BucketId(16, 5)); cmd->setAddress(address); @@ -690,7 +692,7 @@ VisitorManagerTest::testVisitorCleanup() std::ostringstream ost; ost << "testvis" << i; std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("InvalidVisitor", ost.str(), "")); + new api::CreateVisitorCommand(makeBucketSpace(), "InvalidVisitor", ost.str(), "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(0); @@ -703,7 +705,7 @@ VisitorManagerTest::testVisitorCleanup() std::ostringstream ost; ost << "testvis" << (i + 10); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", ost.str(), "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", ost.str(), "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(0); @@ -767,7 +769,7 @@ VisitorManagerTest::testVisitorCleanup() std::ostringstream ost; ost << "testvis" << (i + 24); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", ost.str(), "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", ost.str(), "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(0); @@ -798,7 +800,7 @@ VisitorManagerTest::testAbortOnFailedVisitorInfo() { std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(0); @@ -863,7 +865,7 @@ VisitorManagerTest::testAbortOnFieldPathError() // Use bogus field path to force error to happen std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "testdoctype1.headerval{bogus} == 1234")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); @@ -885,7 +887,7 @@ VisitorManagerTest::testVisitorQueueTimeout() vespalib::MonitorGuard guard(_manager->getThread(0).getQueueMonitor()); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(1); @@ -918,7 +920,7 @@ VisitorManagerTest::testVisitorProcessingTimeout() api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(0); @@ -955,7 +957,7 @@ namespace { ost << "testvis" << ++nextVisitor; api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand("DumpVisitor", ost.str(), "")); + new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", ost.str(), "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(timeout); diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 8abe7a3857d..00051d64b90 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -4,6 +4,7 @@ #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/rawfieldvalue.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> @@ -20,6 +21,7 @@ #include <thread> using namespace std::chrono_literals; +using document::test::makeBucketSpace; namespace storage { @@ -490,7 +492,7 @@ VisitorTest::makeCreateVisitor(const VisitorOptions& options) { api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(options.visitorType, "testvis", "")); + new api::CreateVisitorCommand(makeBucketSpace(), options.visitorType, "testvis", "")); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setMaximumPendingReplyCount(UINT32_MAX); diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp index ce7f7afc670..2a056949f9a 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp @@ -14,6 +14,7 @@ LOG_SETUP(".distributor.bucketdb.updater"); using storage::lib::Node; using storage::lib::NodeType; +using document::BucketSpace; namespace storage::distributor { @@ -81,7 +82,7 @@ BucketDBUpdater::sendRequestBucketInfo( buckets.push_back(bucket); std::shared_ptr<api::RequestBucketInfoCommand> msg( - new api::RequestBucketInfoCommand(buckets)); + new api::RequestBucketInfoCommand(BucketSpace::placeHolder(), buckets)); LOG(debug, "Sending request bucket info command %lu for " diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp index 20a804180f5..b71e2728f5b 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp @@ -100,7 +100,7 @@ GetOperation::sendForChecksum(DistributorMessageSender& sender, const int best = findBestUnsentTarget(res); if (best != -1) { - document::Bucket bucket(BucketSpace::placeHolder(), id); + document::Bucket bucket(_msg->getBucket().getBucketSpace(), id); std::shared_ptr<api::GetCommand> command( std::make_shared<api::GetCommand>( bucket, diff --git a/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.cpp index a3fb083890b..19c693e2a7f 100644 --- a/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.cpp @@ -192,7 +192,7 @@ MultiOperationOperation::onStart(DistributorMessageSender& sender) } assert(blockSize > 4); - document::Bucket bucket(BucketSpace::placeHolder(), bucketIt->first); + document::Bucket bucket(_msg->getBucket().getBucketSpace(), bucketIt->first); //now create a MultiOperationCommand with the new DocumentList std::shared_ptr<api::MultiOperationCommand> command(new api::MultiOperationCommand( diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp index 9b6390c149a..b1fa0d98bb0 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp @@ -116,7 +116,7 @@ PutOperation::checkCreateBucket(const lib::Distribution& dist, // Send create buckets for all nodes in ideal state where we don't // currently have copies. for (uint32_t i = 0; i < createNodes.size(); i++) { - document::Bucket bucket(BucketSpace::placeHolder(), entry.getBucketId()); + document::Bucket bucket(originalCommand.getBucket().getBucketSpace(), entry.getBucketId()); std::shared_ptr<api::CreateBucketCommand> cbc( new api::CreateBucketCommand(bucket)); if (active.contains(createNodes[i])) { @@ -207,7 +207,7 @@ PutOperation::insertDatabaseEntryAndScheduleCreateBucket( } for (uint32_t i=0, n=copies.size(); i<n; ++i) { if (!copies[i].isNewCopy()) continue; - document::Bucket bucket(BucketSpace::placeHolder(), copies[i].getBucketId()); + document::Bucket bucket(originalCommand.getBucket().getBucketSpace(), copies[i].getBucketId()); std::shared_ptr<api::CreateBucketCommand> cbc( new api::CreateBucketCommand(bucket)); if (setOneActive && active.contains(copies[i].getNode().getIndex())) { diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp index bd14ded3710..9ad803b7b7f 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp @@ -10,6 +10,8 @@ #include <vespa/log/log.h> LOG_SETUP(".pendingclusterstate"); +using document::BucketSpace; + namespace storage::distributor { using lib::Node; @@ -317,6 +319,7 @@ PendingClusterState::requestNode(uint16_t node) std::shared_ptr<api::RequestBucketInfoCommand> cmd( new api::RequestBucketInfoCommand( + BucketSpace::placeHolder(), _sender.getDistributorIndex(), _newClusterState, distributionHash)); diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index acfa07affc7..ddc11e9ad77 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -71,7 +71,8 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_CREATEVISITOR: { documentapi::CreateVisitorMessage& from(static_cast<documentapi::CreateVisitorMessage&>(fromMsg)); - auto to = std::make_unique<api::CreateVisitorCommand>(from.getLibraryName(), from.getInstanceId(), + auto to = std::make_unique<api::CreateVisitorCommand>(BucketSpace::placeHolder(), + from.getLibraryName(), from.getInstanceId(), from.getDocumentSelection()); to->setControlDestination(from.getControlDestination()); diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 1c156ff187a..4436312032f 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -387,6 +387,7 @@ public: void setMemoryManager(framework::MemoryManagerInterface& mm) { _memoryManager = &mm; } void setOwnNodeIndex(uint16_t nodeIndex) { _ownNodeIndex = nodeIndex; } + void setBucketSpace(document::BucketSpace bucketSpace) { _bucketSpace = bucketSpace; } const documentapi::LoadType& getLoadType() const { return _initiatingCmd->getLoadType(); diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index 8056d7e0ad2..d3fed86b741 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -497,6 +497,7 @@ VisitorThread::onCreateVisitor( visitor->setDocBlockTimeout(_defaultDocBlockTimeout); visitor->setVisitorInfoTimeout(_defaultVisitorInfoTimeout); visitor->setOwnNodeIndex(_component.getIndex()); + visitor->setBucketSpace(cmd->getBucketSpace()); // Parse document selection try{ diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index bfd657c2f05..cb0cf756586 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -16,6 +16,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/update/fieldpathupdates.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/util/growablebytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> @@ -30,6 +31,7 @@ using document::DocumentId; using document::DocumentType; using document::DocumentTypeRepo; using document::test::makeDocumentBucket; +using document::test::makeBucketSpace; using storage::lib::ClusterState; using vespalib::string; @@ -389,7 +391,7 @@ StorageProtocolTest::testRequestBucketInfo51() std::vector<document::BucketId> ids; ids.push_back(document::BucketId(3)); ids.push_back(document::BucketId(7)); - RequestBucketInfoCommand::SP cmd(new RequestBucketInfoCommand(ids)); + RequestBucketInfoCommand::SP cmd(new RequestBucketInfoCommand(makeBucketSpace(), ids)); RequestBucketInfoCommand::SP cmd2(copyCommand(cmd, _version5_1)); CPPUNIT_ASSERT_EQUAL(ids, cmd2->getBuckets()); CPPUNIT_ASSERT(!cmd2->hasSystemState()); @@ -399,6 +401,7 @@ StorageProtocolTest::testRequestBucketInfo51() { ClusterState state("distributor:3 .1.s:d"); RequestBucketInfoCommand::SP cmd(new RequestBucketInfoCommand( + makeBucketSpace(), 3, state, "14")); RequestBucketInfoCommand::SP cmd2(copyCommand(cmd, _version5_1)); CPPUNIT_ASSERT(cmd2->hasSystemState()); @@ -657,7 +660,7 @@ StorageProtocolTest::testCreateVisitor51() buckets.push_back(document::BucketId(16, 2)); CreateVisitorCommand::SP cmd( - new CreateVisitorCommand("library", "id", "doc selection")); + new CreateVisitorCommand(makeBucketSpace(), "library", "id", "doc selection")); cmd->setControlDestination("controldest"); cmd->setDataDestination("datadest"); cmd->setVisitorCmdId(1); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index d207b307593..e4993a1ee7b 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -562,11 +562,12 @@ ProtocolSerialization4_2::onEncode( api::StorageCommand::UP ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const { + BucketSpace bucketSpace(BucketSpace::placeHolder()); vespalib::stringref libraryName = SH::getString(buf); vespalib::stringref instanceId = SH::getString(buf); vespalib::stringref selection = SH::getString(buf); api::CreateVisitorCommand::UP msg( - new api::CreateVisitorCommand(libraryName, instanceId, selection)); + new api::CreateVisitorCommand(bucketSpace, libraryName, instanceId, selection)); msg->setVisitorCmdId(SH::getInt(buf)); msg->setControlDestination(SH::getString(buf)); msg->setDataDestination(SH::getString(buf)); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp index 2bd160b1aab..22f58ebc58b 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp @@ -638,12 +638,13 @@ ProtocolSerialization5_0::onDecodeRequestBucketInfoCommand(BBuf& buf) const buckets[i] = document::BucketId(SH::getLong(buf)); } api::RequestBucketInfoCommand::UP msg; + BucketSpace bucketSpace(BucketSpace::placeHolder()); if (buckets.size() != 0) { - msg.reset(new api::RequestBucketInfoCommand(buckets)); + msg.reset(new api::RequestBucketInfoCommand(bucketSpace, buckets)); } else { int distributor = SH::getShort(buf); lib::ClusterState state(SH::getString(buf)); - msg.reset(new api::RequestBucketInfoCommand(distributor, state, SH::getString(buf))); + msg.reset(new api::RequestBucketInfoCommand(bucketSpace, distributor, state, SH::getString(buf))); } onDecodeCommand(buf, *msg); return api::StorageCommand::UP(msg.release()); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp index 4f84eff0ab8..dc97742b733 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp @@ -161,11 +161,12 @@ ProtocolSerialization5_1::onEncode( api::StorageCommand::UP ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const { + BucketSpace bucketSpace(BucketSpace::placeHolder()); vespalib::stringref libraryName = SH::getString(buf); vespalib::stringref instanceId = SH::getString(buf); vespalib::stringref selection = SH::getString(buf); api::CreateVisitorCommand::UP msg( - new api::CreateVisitorCommand(libraryName, instanceId, selection)); + new api::CreateVisitorCommand(bucketSpace, libraryName, instanceId, selection)); msg->setVisitorCmdId(SH::getInt(buf)); msg->setControlDestination(SH::getString(buf)); msg->setDataDestination(SH::getString(buf)); diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp index 983a65114b3..18ad95c2c02 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.cpp +++ b/storageapi/src/vespa/storageapi/message/bucket.cpp @@ -440,8 +440,10 @@ ApplyBucketDiffReply::print(std::ostream& out, bool verbose, } RequestBucketInfoCommand::RequestBucketInfoCommand( + document::BucketSpace bucketSpace, const std::vector<document::BucketId>& buckets) : StorageCommand(MessageType::REQUESTBUCKETINFO), + _bucketSpace(bucketSpace), _buckets(buckets), _state(), _distributor(0xFFFF) @@ -449,9 +451,11 @@ RequestBucketInfoCommand::RequestBucketInfoCommand( } RequestBucketInfoCommand::RequestBucketInfoCommand( + document::BucketSpace bucketSpace, uint16_t distributor, const lib::ClusterState& state, const vespalib::stringref & distributionHash) : StorageCommand(MessageType::REQUESTBUCKETINFO), + _bucketSpace(bucketSpace), _buckets(), _state(new lib::ClusterState(state)), _distributor(distributor), @@ -460,8 +464,10 @@ RequestBucketInfoCommand::RequestBucketInfoCommand( } RequestBucketInfoCommand::RequestBucketInfoCommand( + document::BucketSpace bucketSpace, uint16_t distributor, const lib::ClusterState& state) : StorageCommand(MessageType::REQUESTBUCKETINFO), + _bucketSpace(bucketSpace), _buckets(), _state(new lib::ClusterState(state)), _distributor(distributor), diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h index d2bf97e7768..0c97772d184 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.h +++ b/storageapi/src/vespa/storageapi/message/bucket.h @@ -336,6 +336,7 @@ public: * the buckets that belong to the given distributor should be returned. */ class RequestBucketInfoCommand : public StorageCommand { + document::BucketSpace _bucketSpace; std::vector<document::BucketId> _buckets; std::unique_ptr<lib::ClusterState> _state; uint16_t _distributor; @@ -343,12 +344,15 @@ class RequestBucketInfoCommand : public StorageCommand { public: explicit RequestBucketInfoCommand( + document::BucketSpace bucketSpace, const std::vector<document::BucketId>& buckets); - RequestBucketInfoCommand(uint16_t distributor, + RequestBucketInfoCommand(document::BucketSpace bucketSpace, + uint16_t distributor, const lib::ClusterState& state, const vespalib::stringref & _distributionHash); - RequestBucketInfoCommand(uint16_t distributor, + RequestBucketInfoCommand(document::BucketSpace bucketSpace, + uint16_t distributor, const lib::ClusterState& state); const std::vector<document::BucketId>& getBuckets() const { return _buckets; } diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp index 8c10eb501b2..7b5a614bd3e 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ b/storageapi/src/vespa/storageapi/message/visitor.cpp @@ -13,10 +13,12 @@ IMPLEMENT_REPLY(DestroyVisitorReply) IMPLEMENT_COMMAND(VisitorInfoCommand, VisitorInfoReply) IMPLEMENT_REPLY(VisitorInfoReply) -CreateVisitorCommand::CreateVisitorCommand(const vespalib::stringref & libraryName, +CreateVisitorCommand::CreateVisitorCommand(document::BucketSpace bucketSpace, + const vespalib::stringref & libraryName, const vespalib::stringref & instanceId, const vespalib::stringref & docSelection) : StorageCommand(MessageType::VISITOR_CREATE), + _bucketSpace(bucketSpace), _libName(libraryName), _params(), _controlDestination(), @@ -41,6 +43,7 @@ CreateVisitorCommand::CreateVisitorCommand(const vespalib::stringref & libraryNa CreateVisitorCommand::CreateVisitorCommand(const CreateVisitorCommand& o) : StorageCommand(o), + _bucketSpace(o._bucketSpace), _libName(o._libName), _params(o._params), _controlDestination(o._controlDestination), diff --git a/storageapi/src/vespa/storageapi/message/visitor.h b/storageapi/src/vespa/storageapi/message/visitor.h index 3d659831351..f252ecd344f 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.h +++ b/storageapi/src/vespa/storageapi/message/visitor.h @@ -26,6 +26,7 @@ namespace api { */ class CreateVisitorCommand : public StorageCommand { private: + document::BucketSpace _bucketSpace; vespalib::string _libName; // Name of visitor library to use, ie. DumpVisitor.so vdslib::Parameters _params; @@ -53,7 +54,8 @@ private: uint32_t _maxBucketsPerVisitor; public: - CreateVisitorCommand(const vespalib::stringref & libraryName, + CreateVisitorCommand(document::BucketSpace bucketSpace, + const vespalib::stringref & libraryName, const vespalib::stringref & instanceId, const vespalib::stringref & docSelection); @@ -83,6 +85,7 @@ public: VisitorId getVisitorId() const { return _visitorId; } uint32_t getVisitorCmdId() const { return _visitorCmdId; } + document::BucketSpace getBucketSpace() const { return _bucketSpace; } const vespalib::string & getLibraryName() const { return _libName; } const vespalib::string & getInstanceId() const { return _instanceId; } const vespalib::string & getControlDestination() const |