diff options
Diffstat (limited to 'storage/src/tests/persistence/filestorage/filestormanagertest.cpp')
-rw-r--r-- | storage/src/tests/persistence/filestorage/filestormanagertest.cpp | 109 |
1 files changed, 56 insertions, 53 deletions
diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 91ccf2b123f..2b0218bf20c 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -1,10 +1,11 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/dummystoragelink.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <tests/persistence/filestorage/forwardingmessagesender.h> #include <vespa/config/common/exceptions.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/select/parser.h> @@ -65,11 +66,11 @@ namespace storage { namespace { -vespalib::string _Cluster("cluster"); -vespalib::string _Storage("storage"); -api::StorageMessageAddress _Storage2(&_Storage, lib::NodeType::STORAGE, 2); -api::StorageMessageAddress _Storage3(&_Storage, lib::NodeType::STORAGE, 3); -api::StorageMessageAddress _Cluster1(&_Cluster, lib::NodeType::STORAGE, 1); +vespalib::string _cluster("cluster"); +vespalib::string _storage("storage"); +api::StorageMessageAddress _storage2(&_storage, lib::NodeType::STORAGE, 2); +api::StorageMessageAddress _storage3(&_storage, lib::NodeType::STORAGE, 3); +api::StorageMessageAddress _cluster1(&_cluster, lib::NodeType::STORAGE, 1); struct TestFileStorComponents; @@ -93,7 +94,7 @@ struct FileStorTestBase : Test { const document::DocumentType* _testdoctype1; FileStorTestBase() : _node(), _waitTime(LONG_WAITTIME) {} - ~FileStorTestBase(); + ~FileStorTestBase() override; void SetUp() override; void TearDown() override; @@ -223,8 +224,10 @@ struct TestFileStorComponents { explicit TestFileStorComponents(FileStorTestBase& test, bool use_small_config = false) : manager(nullptr) { - auto fsm = std::make_unique<FileStorManager>(config::ConfigUri((use_small_config ? test.smallConfig : test.config)->getConfigId()), - test._node->getPersistenceProvider(), + using vespa::config::content::StorFilestorConfig; + auto config_uri = config::ConfigUri((use_small_config ? test.smallConfig : test.config)->getConfigId()); + auto config = config_from<StorFilestorConfig>(config_uri); + auto fsm = std::make_unique<FileStorManager>(*config, test._node->getPersistenceProvider(), test._node->getComponentRegister(), *test._node, test._node->get_host_info()); manager = fsm.get(); top.push_back(std::move(fsm)); @@ -324,7 +327,7 @@ TEST_F(FileStorManagerTest, header_only_put) { // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -339,7 +342,7 @@ TEST_F(FileStorManagerTest, header_only_put) { { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 124); cmd->setUpdateTimestamp(105); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -351,7 +354,7 @@ TEST_F(FileStorManagerTest, header_only_put) { // Getting it { auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(bid), doc->getId(), document::AllFields::NAME); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -386,7 +389,7 @@ TEST_F(FileStorManagerTest, put) { // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -467,7 +470,7 @@ TEST_F(FileStorManagerTest, flush) { std::vector<std::shared_ptr<api::StorageCommand> > _commands; for (uint32_t i=0; i<msgCount; ++i) { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, i+1); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); _commands.push_back(cmd); } for (uint32_t i=0; i<msgCount; ++i) { @@ -494,7 +497,7 @@ TEST_F(FileStorManagerTest, handler_priority) { // Populate bucket with the given data for (uint32_t i = 1; i < 6; i++) { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(i * 15); filestorHandler.schedule(cmd); } @@ -629,7 +632,7 @@ TEST_F(FileStorManagerTest, handler_pause) { // Populate bucket with the given data for (uint32_t i = 1; i < 6; i++) { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(i * 15); filestorHandler.schedule(cmd); } @@ -705,7 +708,7 @@ TEST_F(FileStorManagerTest, handler_timeout) { // Populate bucket with the given data { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(0); cmd->setTimeout(50ms); filestorHandler.schedule(cmd); @@ -713,7 +716,7 @@ TEST_F(FileStorManagerTest, handler_timeout) { { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(200); cmd->setTimeout(10000ms); filestorHandler.schedule(cmd); @@ -773,7 +776,7 @@ TEST_F(FileStorManagerTest, priority) { document::BucketId bucket(16, factory.getBucketId(documents[i]->getId()).getRawId()); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), documents[i], 100 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(i * 2); filestorHandler.schedule(cmd); } @@ -832,7 +835,7 @@ TEST_F(FileStorManagerTest, split1) { _node->getPersistenceProvider().createBucket(makeSpiBucket(bucket)); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), documents[i], 100 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setSourceIndex(0); filestorHandler.schedule(cmd); @@ -847,7 +850,7 @@ TEST_F(FileStorManagerTest, split1) { // Delete every 5th document to have delete entries in file too if (i % 5 == 0) { auto rcmd = std::make_shared<api::RemoveCommand>(makeDocumentBucket(bucket), documents[i]->getId(), 1000000 + 100 + i); - rcmd->setAddress(_Storage3); + rcmd->setAddress(_storage3); filestorHandler.schedule(rcmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -875,7 +878,7 @@ TEST_F(FileStorManagerTest, split1) { for (uint32_t i=0; i<documents.size(); ++i) { document::BucketId bucket(17, i % 3 == 0 ? 0x10001 : 0x0100001); auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -907,7 +910,7 @@ TEST_F(FileStorManagerTest, split1) { bucket = document::BucketId(33, factory.getBucketId(documents[i]->getId()).getRawId()); } auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -953,7 +956,7 @@ TEST_F(FileStorManagerTest, split_single_group) { _node->getPersistenceProvider().createBucket(makeSpiBucket(bucket)); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), documents[i], 100 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -979,7 +982,7 @@ TEST_F(FileStorManagerTest, split_single_group) { for (uint32_t i=0; i<documents.size(); ++i) { document::BucketId bucket(17, state ? 0x10001 : 0x00001); auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -1007,7 +1010,7 @@ FileStorTestBase::putDoc(DummyStorageLink& top, _node->getPersistenceProvider().createBucket(makeSpiBucket(target)); auto doc = std::make_shared<Document>(*_node->getTypeRepo(), *_testdoctype1, docId); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(target), doc, docNum+1); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(120); filestorHandler.schedule(cmd); filestorHandler.flush(true); @@ -1048,7 +1051,7 @@ TEST_F(FileStorManagerTest, split_empty_target_with_remapped_ops) { vespalib::make_string("id:ns:testdoctype1:n=%d:1234", 0x100001)); auto doc = std::make_shared<Document>(*_node->getTypeRepo(), *_testdoctype1, docId); auto putCmd = std::make_shared<api::PutCommand>(makeDocumentBucket(source), doc, 1001); - putCmd->setAddress(_Storage3); + putCmd->setAddress(_storage3); putCmd->setPriority(120); filestorHandler.schedule(splitCmd); @@ -1129,7 +1132,7 @@ TEST_F(FileStorManagerTest, join) { for (uint32_t i=0; i<documents.size(); ++i) { document::BucketId bucket(17, factory.getBucketId(documents[i]->getId()).getRawId()); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), documents[i], 100 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -1141,7 +1144,7 @@ TEST_F(FileStorManagerTest, join) { if ((i % 5) == 0) { auto rcmd = std::make_shared<api::RemoveCommand>( makeDocumentBucket(bucket), documents[i]->getId(), 1000000 + 100 + i); - rcmd->setAddress(_Storage3); + rcmd->setAddress(_storage3); filestorHandler.schedule(rcmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -1170,7 +1173,7 @@ TEST_F(FileStorManagerTest, join) { document::BucketId bucket(16, 1); auto cmd = std::make_shared<api::GetCommand>( makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -1342,7 +1345,7 @@ TEST_F(FileStorManagerTest, remove_location) { docid << "id:ns:testdoctype1:n=" << (i << 8) << ":foo"; Document::SP doc(createDocument("some content", docid.str())); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 1000 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1355,7 +1358,7 @@ TEST_F(FileStorManagerTest, remove_location) { // Issuing remove location command { auto cmd = std::make_shared<api::RemoveLocationCommand>("id.user % 512 == 0", makeDocumentBucket(bid)); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1381,7 +1384,7 @@ TEST_F(FileStorManagerTest, delete_bucket) { // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1397,7 +1400,7 @@ TEST_F(FileStorManagerTest, delete_bucket) { // Delete bucket { auto cmd = std::make_shared<api::DeleteBucketCommand>(makeDocumentBucket(bid)); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); cmd->setBucketInfo(bucketInfo); top.sendDown(cmd); top.waitForMessages(1, _waitTime); @@ -1423,7 +1426,7 @@ TEST_F(FileStorManagerTest, delete_bucket_rejects_outdated_bucket_info) { // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1440,7 +1443,7 @@ TEST_F(FileStorManagerTest, delete_bucket_rejects_outdated_bucket_info) { { auto cmd = std::make_shared<api::DeleteBucketCommand>(makeDocumentBucket(bid)); cmd->setBucketInfo(api::BucketInfo(0xf000baaa, 1, 123, 1, 456)); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1468,7 +1471,7 @@ TEST_F(FileStorManagerTest, delete_bucket_with_invalid_bucket_info){ // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1482,7 +1485,7 @@ TEST_F(FileStorManagerTest, delete_bucket_with_invalid_bucket_info){ // Attempt to delete bucket with invalid bucketinfo { auto cmd = std::make_shared<api::DeleteBucketCommand>(makeDocumentBucket(bid)); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1506,7 +1509,7 @@ TEST_F(FileStorManagerTest, no_timestamps) { // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 0); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); EXPECT_EQ(api::Timestamp(0), cmd->getTimestamp()); top.sendDown(cmd); top.waitForMessages(1, _waitTime); @@ -1519,7 +1522,7 @@ TEST_F(FileStorManagerTest, no_timestamps) { // Removing it { auto cmd = std::make_shared<api::RemoveCommand>(makeDocumentBucket(bid), doc->getId(), 0); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); EXPECT_EQ(api::Timestamp(0), cmd->getTimestamp()); top.sendDown(cmd); top.waitForMessages(1, _waitTime); @@ -1544,7 +1547,7 @@ TEST_F(FileStorManagerTest, equal_timestamps) { Document::SP doc(createDocument( "some content", "id:crawler:testdoctype1:n=4000:http://www.ntnu.no/")); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1561,7 +1564,7 @@ TEST_F(FileStorManagerTest, equal_timestamps) { Document::SP doc(createDocument( "some content", "id:crawler:testdoctype1:n=4000:http://www.ntnu.no/")); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1576,7 +1579,7 @@ TEST_F(FileStorManagerTest, equal_timestamps) { Document::SP doc(createDocument( "some content", "id:crawler:testdoctype1:n=4000:http://www.ntnu.nu/")); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1608,7 +1611,7 @@ TEST_F(FileStorManagerTest, get_iter) { // Putting all docs to have something to visit for (uint32_t i=0; i<docs.size(); ++i) { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), docs[i], 100 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1671,7 +1674,7 @@ TEST_F(FileStorManagerTest, set_bucket_active_state) { { auto cmd = std::make_shared<api::SetBucketStateCommand>(makeDocumentBucket(bid), api::SetBucketStateCommand::ACTIVE); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1690,7 +1693,7 @@ TEST_F(FileStorManagerTest, set_bucket_active_state) { { auto cmd = std::make_shared<api::SetBucketStateCommand>( makeDocumentBucket(bid), api::SetBucketStateCommand::INACTIVE); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1717,7 +1720,7 @@ TEST_F(FileStorManagerTest, notify_owner_distributor_on_outdated_set_bucket_stat createBucket(bid); auto cmd = std::make_shared<api::SetBucketStateCommand>(makeDocumentBucket(bid), api::SetBucketStateCommand::ACTIVE); - cmd->setAddress(_Cluster1); + cmd->setAddress(_cluster1); cmd->setSourceIndex(0); top.sendDown(cmd); @@ -1752,7 +1755,7 @@ TEST_F(FileStorManagerTest, GetBucketDiff_implicitly_creates_bucket) { std::vector<api::MergeBucketCommand::Node> nodes = {1, 0}; auto cmd = std::make_shared<api::GetBucketDiffCommand>(makeDocumentBucket(bid), nodes, Timestamp(1000)); - cmd->setAddress(_Cluster1); + cmd->setAddress(_cluster1); cmd->setSourceIndex(0); top.sendDown(cmd); @@ -1776,7 +1779,7 @@ TEST_F(FileStorManagerTest, merge_bucket_implicitly_creates_bucket) { std::vector<api::MergeBucketCommand::Node> nodes = {1, 2}; auto cmd = std::make_shared<api::MergeBucketCommand>(makeDocumentBucket(bid), nodes, Timestamp(1000)); - cmd->setAddress(_Cluster1); + cmd->setAddress(_cluster1); cmd->setSourceIndex(0); top.sendDown(cmd); @@ -1797,7 +1800,7 @@ TEST_F(FileStorManagerTest, newly_created_bucket_is_ready) { document::BucketId bid(16, 4000); auto cmd = std::make_shared<api::CreateBucketCommand>(makeDocumentBucket(bid)); - cmd->setAddress(_Cluster1); + cmd->setAddress(_cluster1); cmd->setSourceIndex(0); top.sendDown(cmd); @@ -1818,7 +1821,7 @@ TEST_F(FileStorManagerTest, create_bucket_sets_active_flag_in_database_and_reply document::BucketId bid(16, 4000); auto cmd = std::make_shared<api::CreateBucketCommand>(makeDocumentBucket(bid)); - cmd->setAddress(_Cluster1); + cmd->setAddress(_cluster1); cmd->setSourceIndex(0); cmd->setActive(true); c.top.sendDown(cmd); @@ -1837,7 +1840,7 @@ TEST_F(FileStorManagerTest, create_bucket_sets_active_flag_in_database_and_reply template <typename Metric> void FileStorTestBase::assert_request_size_set(TestFileStorComponents& c, std::shared_ptr<api::StorageMessage> cmd, const Metric& metric) { cmd->setApproxByteSize(54321); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); c.top.sendDown(cmd); c.top.waitForMessages(1, _waitTime); EXPECT_EQ(static_cast<int64_t>(cmd->getApproxByteSize()), metric.request_size.getLast()); @@ -1894,7 +1897,7 @@ TEST_F(FileStorManagerTest, test_and_set_condition_mismatch_not_counted_as_failu createBucket(bucket); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), std::move(doc), api::Timestamp(12345)); cmd->setCondition(TestAndSetCondition("not testdoctype1")); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); c.top.sendDown(cmd); api::PutReply* reply; |