diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-10-14 23:07:22 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2020-10-15 04:59:11 +0000 |
commit | 30d76ff5080f3f911d9119125202d7bad0a2a9da (patch) | |
tree | 9123279b9637e8cbab0b116428892e8990215d48 /storage/src/tests/persistence | |
parent | f07e7cde693a73d99d6d3d27dc3aa65e44d1958b (diff) |
GC disk related code.
Diffstat (limited to 'storage/src/tests/persistence')
12 files changed, 157 insertions, 268 deletions
diff --git a/storage/src/tests/persistence/common/filestortestfixture.cpp b/storage/src/tests/persistence/common/filestortestfixture.cpp index 1282bcf85c3..49dbf082e34 100644 --- a/storage/src/tests/persistence/common/filestortestfixture.cpp +++ b/storage/src/tests/persistence/common/filestortestfixture.cpp @@ -28,8 +28,7 @@ FileStorTestFixture::setupPersistenceThreads(uint32_t threads) _config->getConfig("stor-server").set("node_index", "1"); _config->getConfig("stor-filestor").set("num_threads", std::to_string(threads)); - _node = std::make_unique<TestServiceLayerApp>( - DiskCount(1), NodeIndex(1), _config->getConfigId()); + _node = std::make_unique<TestServiceLayerApp>(NodeIndex(1), _config->getConfigId()); _testdoctype1 = _node->getTypeRepo()->getDocumentType("testdoctype1"); } @@ -60,7 +59,6 @@ FileStorTestFixture::createBucket(const document::BucketId& bid) StorBucketDatabase::WrappedEntry entry( _node->getStorageBucketDatabase().get(bid, "foo", StorBucketDatabase::CREATE_IF_NONEXISTING)); - entry->disk = 0; entry->info = api::BucketInfo(0, 0, 0, 0, 0, true, false); entry.write(); } diff --git a/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp b/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp index 18f8a235453..0fe18335c23 100644 --- a/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp +++ b/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp @@ -40,7 +40,6 @@ TEST_F(DeactivateBucketsTest, buckets_in_database_deactivated_when_node_down_in_ StorBucketDatabase::WrappedEntry entry( _node->getStorageBucketDatabase().get(bucket, "foo", StorBucketDatabase::CREATE_IF_NONEXISTING)); - entry->disk = 0; entry->info = serviceLayerInfo; entry.write(); } diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index b7165312785..3525563eb7a 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -82,7 +82,6 @@ struct FileStorManagerTest : Test{ StorBucketDatabase::WrappedEntry entry( _node->getStorageBucketDatabase().get(bid, "foo", StorBucketDatabase::CREATE_IF_NONEXISTING)); - entry->disk = disk; entry->info = api::BucketInfo(0, 0, 0, 0, 0, true, false); entry.write(); } @@ -123,7 +122,7 @@ struct FileStorManagerTest : Test{ new lib::ClusterState(state))); } - void setupDisks(uint32_t diskCount) { + void setupDisks() { std::string rootOfRoot = "filestormanagertest"; config.reset(new vdstestlib::DirConfig(getStandardConfig(true, rootOfRoot))); @@ -144,8 +143,7 @@ struct FileStorManagerTest : Test{ assert(system(vespalib::make_string("mkdir -p %s/disks/d0", getRootFolder(*config).c_str()).c_str()) == 0); assert(system(vespalib::make_string("mkdir -p %s/disks/d0", getRootFolder(*config2).c_str()).c_str()) == 0); try { - _node.reset(new TestServiceLayerApp(DiskCount(diskCount), NodeIndex(0), - config->getConfigId())); + _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), config->getConfigId()); _node->setupDummyPersistence(); } catch (config::InvalidConfigException& e) { fprintf(stderr, "%s\n", e.what()); @@ -198,12 +196,11 @@ std::unique_ptr<DiskThread> createThread(vdstestlib::DirConfig& config, TestServiceLayerApp& node, spi::PersistenceProvider& provider, FileStorHandler& filestorHandler, - FileStorThreadMetrics& metrics, - uint16_t deviceIndex) + FileStorThreadMetrics& metrics) { (void) config; return std::make_unique<PersistenceThread>(nullptr,node.getComponentRegister(), config.getConfigId(), - provider, filestorHandler, metrics, deviceIndex); + provider, filestorHandler, metrics); } namespace { @@ -227,7 +224,7 @@ struct TestFileStorComponents { void FileStorManagerTest::SetUp() { - setupDisks(1); + setupDisks(); } void @@ -399,7 +396,7 @@ TEST_F(FileStorManagerTest, handler_priority) { FileStorHandler filestorHandler(messageSender, metrics, _node->getComponentRegister()); filestorHandler.setGetNextMessageTimeout(50ms); - uint32_t stripeId = filestorHandler.getNextStripeId(0); + uint32_t stripeId = filestorHandler.getNextStripeId(); ASSERT_EQ(0u, stripeId); std::string content("Here is some content which is in all documents"); @@ -416,14 +413,14 @@ TEST_F(FileStorManagerTest, handler_priority) { auto address = std::make_shared<api::StorageMessageAddress>("storage", lib::NodeType::STORAGE, 3); cmd->setAddress(*address); cmd->setPriority(i * 15); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); } - ASSERT_EQ(15, filestorHandler.getNextMessage(0, stripeId).second->getPriority()); - ASSERT_EQ(30, filestorHandler.getNextMessage(0, stripeId).second->getPriority()); - ASSERT_EQ(45, filestorHandler.getNextMessage(0, stripeId).second->getPriority()); - ASSERT_EQ(60, filestorHandler.getNextMessage(0, stripeId).second->getPriority()); - ASSERT_EQ(75, filestorHandler.getNextMessage(0, stripeId).second->getPriority()); + ASSERT_EQ(15, filestorHandler.getNextMessage(stripeId).second->getPriority()); + ASSERT_EQ(30, filestorHandler.getNextMessage(stripeId).second->getPriority()); + ASSERT_EQ(45, filestorHandler.getNextMessage(stripeId).second->getPriority()); + ASSERT_EQ(60, filestorHandler.getNextMessage(stripeId).second->getPriority()); + ASSERT_EQ(75, filestorHandler.getNextMessage(stripeId).second->getPriority()); } class MessagePusherThread : public document::Runnable { @@ -442,7 +439,7 @@ public: document::BucketId bucket(16, factory.getBucketId(_doc->getId()).getRawId()); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), _doc, 100); - _handler.schedule(cmd, 0); + _handler.schedule(cmd); std::this_thread::sleep_for(1ms); } @@ -466,13 +463,13 @@ public: std::atomic<bool> _threadDone; explicit MessageFetchingThread(FileStorHandler& handler) - : _threadId(handler.getNextStripeId(0)), _handler(handler), _config(0), _fetchedCount(0), _done(false), + : _threadId(handler.getNextStripeId()), _handler(handler), _config(0), _fetchedCount(0), _done(false), _failed(false), _threadDone(false) {} void run() override { while (!_done) { - FileStorHandler::LockedMessage msg = _handler.getNextMessage(0, _threadId); + FileStorHandler::LockedMessage msg = _handler.getNextMessage(_threadId); if (msg.second.get()) { uint32_t originalConfig = _config.load(); _fetchedCount++; @@ -553,7 +550,7 @@ TEST_F(FileStorManagerTest, handler_pause) { FileStorHandler filestorHandler(messageSender, metrics, _node->getComponentRegister()); filestorHandler.setGetNextMessageTimeout(50ms); - uint32_t stripeId = filestorHandler.getNextStripeId(0); + uint32_t stripeId = filestorHandler.getNextStripeId(); std::string content("Here is some content which is in all documents"); std::ostringstream uri; @@ -569,18 +566,18 @@ TEST_F(FileStorManagerTest, handler_pause) { auto address = std::make_unique<api::StorageMessageAddress>("storage", lib::NodeType::STORAGE, 3); cmd->setAddress(*address); cmd->setPriority(i * 15); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); } - ASSERT_EQ(15, filestorHandler.getNextMessage(0, stripeId).second->getPriority()); + ASSERT_EQ(15, filestorHandler.getNextMessage(stripeId).second->getPriority()); { ResumeGuard guard = filestorHandler.pause(); (void)guard; - ASSERT_EQ(filestorHandler.getNextMessage(0, stripeId).second.get(), nullptr); + ASSERT_EQ(filestorHandler.getNextMessage(stripeId).second.get(), nullptr); } - ASSERT_EQ(30, filestorHandler.getNextMessage(0, stripeId).second->getPriority()); + ASSERT_EQ(30, filestorHandler.getNextMessage(stripeId).second->getPriority()); } TEST_F(FileStorManagerTest, remap_split) { @@ -612,8 +609,8 @@ TEST_F(FileStorManagerTest, remap_split) { // Populate bucket with the given data for (uint32_t i = 1; i < 4; i++) { - filestorHandler.schedule(std::make_shared<api::PutCommand>(makeDocumentBucket(bucket1), doc1, i), 0); - filestorHandler.schedule(std::make_shared<api::PutCommand>(makeDocumentBucket(bucket2), doc2, i + 10), 0); + filestorHandler.schedule(std::make_shared<api::PutCommand>(makeDocumentBucket(bucket1), doc1, i)); + filestorHandler.schedule(std::make_shared<api::PutCommand>(makeDocumentBucket(bucket2), doc2, i + 10)); } EXPECT_EQ("BucketId(0x40000000000004d2): Put(BucketId(0x40000000000004d2), id:footype:testdoctype1:n=1234:bar, timestamp 1, size 118) (priority: 127)\n" @@ -622,11 +619,11 @@ TEST_F(FileStorManagerTest, remap_split) { "BucketId(0x40000000000011d7): Put(BucketId(0x40000000000011d7), id:footype:testdoctype1:n=4567:bar, timestamp 12, size 118) (priority: 127)\n" "BucketId(0x40000000000004d2): Put(BucketId(0x40000000000004d2), id:footype:testdoctype1:n=1234:bar, timestamp 3, size 118) (priority: 127)\n" "BucketId(0x40000000000011d7): Put(BucketId(0x40000000000011d7), id:footype:testdoctype1:n=4567:bar, timestamp 13, size 118) (priority: 127)\n", - filestorHandler.dumpQueue(0)); + filestorHandler.dumpQueue()); - FileStorHandler::RemapInfo a(makeDocumentBucket(document::BucketId(17, 1234)), 0); - FileStorHandler::RemapInfo b(makeDocumentBucket(document::BucketId(17, 1234 | 1 << 16)), 0); - filestorHandler.remapQueueAfterSplit(FileStorHandler::RemapInfo(makeDocumentBucket(bucket1), 0), a, b); + FileStorHandler::RemapInfo a(makeDocumentBucket(document::BucketId(17, 1234))); + FileStorHandler::RemapInfo b(makeDocumentBucket(document::BucketId(17, 1234 | 1 << 16))); + filestorHandler.remapQueueAfterSplit(FileStorHandler::RemapInfo(makeDocumentBucket(bucket1)), a, b); ASSERT_TRUE(a.foundInQueue); ASSERT_FALSE(b.foundInQueue); @@ -637,7 +634,7 @@ TEST_F(FileStorManagerTest, remap_split) { "BucketId(0x44000000000004d2): Put(BucketId(0x44000000000004d2), id:footype:testdoctype1:n=1234:bar, timestamp 1, size 118) (priority: 127)\n" "BucketId(0x44000000000004d2): Put(BucketId(0x44000000000004d2), id:footype:testdoctype1:n=1234:bar, timestamp 2, size 118) (priority: 127)\n" "BucketId(0x44000000000004d2): Put(BucketId(0x44000000000004d2), id:footype:testdoctype1:n=1234:bar, timestamp 3, size 118) (priority: 127)\n", - filestorHandler.dumpQueue(0)); + filestorHandler.dumpQueue()); } TEST_F(FileStorManagerTest, handler_timeout) { @@ -657,7 +654,7 @@ TEST_F(FileStorManagerTest, handler_timeout) { FileStorHandler filestorHandler(messageSender, metrics, _node->getComponentRegister()); filestorHandler.setGetNextMessageTimeout(50ms); - uint32_t stripeId = filestorHandler.getNextStripeId(0); + uint32_t stripeId = filestorHandler.getNextStripeId(); std::string content("Here is some content which is in all documents"); std::ostringstream uri; @@ -674,7 +671,7 @@ TEST_F(FileStorManagerTest, handler_timeout) { cmd->setAddress(*address); cmd->setPriority(0); cmd->setTimeout(50ms); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); } { @@ -683,12 +680,12 @@ TEST_F(FileStorManagerTest, handler_timeout) { cmd->setAddress(*address); cmd->setPriority(200); cmd->setTimeout(10000ms); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); } std::this_thread::sleep_for(51ms); for (;;) { - auto lock = filestorHandler.getNextMessage(0, stripeId); + auto lock = filestorHandler.getNextMessage(stripeId); if (lock.first.get()) { ASSERT_EQ(200, lock.second->getPriority()); break; @@ -718,10 +715,10 @@ TEST_F(FileStorManagerTest, priority) { FileStorHandler filestorHandler(messageSender, metrics, _node->getComponentRegister()); std::unique_ptr<DiskThread> thread(createThread( *config, *_node, _node->getPersistenceProvider(), - filestorHandler, *metrics.disks[0]->threads[0], 0)); + filestorHandler, *metrics.disks[0]->threads[0])); std::unique_ptr<DiskThread> thread2(createThread( *config, *_node, _node->getPersistenceProvider(), - filestorHandler, *metrics.disks[0]->threads[1], 0)); + filestorHandler, *metrics.disks[0]->threads[1])); // Creating documents to test with. Different gids, 2 locations. std::vector<document::Document::SP > documents; @@ -753,7 +750,7 @@ TEST_F(FileStorManagerTest, priority) { auto address = std::make_unique<api::StorageMessageAddress>("storage", lib::NodeType::STORAGE, 3); cmd->setAddress(*address); cmd->setPriority(i * 2); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); } filestorHandler.flush(true); @@ -798,7 +795,7 @@ TEST_F(FileStorManagerTest, split1) { FileStorHandler filestorHandler(messageSender, metrics, _node->getComponentRegister()); std::unique_ptr<DiskThread> thread(createThread( *config, *_node, _node->getPersistenceProvider(), - filestorHandler, *metrics.disks[0]->threads[0], 0)); + filestorHandler, *metrics.disks[0]->threads[0])); // Creating documents to test with. Different gids, 2 locations. std::vector<document::Document::SP > documents; for (uint32_t i=0; i<20; ++i) { @@ -828,7 +825,7 @@ TEST_F(FileStorManagerTest, split1) { cmd->setAddress(*address); cmd->setSourceIndex(0); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); LOG(debug, "Got %zu replies", top.getNumReplies()); ASSERT_EQ(1, top.getNumReplies()); @@ -842,7 +839,7 @@ TEST_F(FileStorManagerTest, split1) { auto rcmd = std::make_shared<api::RemoveCommand>( makeDocumentBucket(bucket), documents[i]->getId(), 1000000 + 100 + i); rcmd->setAddress(*address); - filestorHandler.schedule(rcmd, 0); + filestorHandler.schedule(rcmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto rreply = std::dynamic_pointer_cast<api::RemoveReply>(top.getReply(0)); @@ -856,7 +853,7 @@ TEST_F(FileStorManagerTest, split1) { { auto cmd = std::make_shared<api::SplitBucketCommand>(makeDocumentBucket(document::BucketId(16, 1))); cmd->setSourceIndex(0); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto reply = std::dynamic_pointer_cast<api::SplitBucketReply>(top.getReply(0)); @@ -873,7 +870,7 @@ TEST_F(FileStorManagerTest, split1) { makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); cmd->setAddress(address); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto reply = std::dynamic_pointer_cast<api::GetReply>(top.getReply(0)); @@ -887,7 +884,7 @@ TEST_F(FileStorManagerTest, split1) { auto cmd = std::make_shared<api::SplitBucketCommand>( makeDocumentBucket(document::BucketId(i, 0x0100001))); cmd->setSourceIndex(0); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto reply = std::dynamic_pointer_cast<api::SplitBucketReply>(top.getReply(0)); @@ -909,7 +906,7 @@ TEST_F(FileStorManagerTest, split1) { makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); cmd->setAddress(address); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto reply = std::dynamic_pointer_cast<api::GetReply>(top.getReply(0)); @@ -945,7 +942,7 @@ TEST_F(FileStorManagerTest, split_single_group) { std::unique_ptr<DiskThread> thread(createThread( *config, *_node, _node->getPersistenceProvider(), - filestorHandler, *metrics.disks[0]->threads[0], 0)); + filestorHandler, *metrics.disks[0]->threads[0])); // Creating documents to test with. Different gids, 2 locations. std::vector<document::Document::SP> documents; for (uint32_t i=0; i<20; ++i) { @@ -968,7 +965,7 @@ TEST_F(FileStorManagerTest, split_single_group) { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), documents[i], 100 + i); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); cmd->setAddress(address); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto reply = std::dynamic_pointer_cast<api::PutReply>(top.getReply(0)); @@ -980,7 +977,7 @@ TEST_F(FileStorManagerTest, split_single_group) { { auto cmd = std::make_shared<api::SplitBucketCommand>(makeDocumentBucket(document::BucketId(16, 1))); cmd->setSourceIndex(0); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto reply = std::dynamic_pointer_cast<api::SplitBucketReply>(top.getReply(0)); @@ -996,7 +993,7 @@ TEST_F(FileStorManagerTest, split_single_group) { (makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); cmd->setAddress(address); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto reply = std::dynamic_pointer_cast<api::GetReply>(top.getReply(0)); @@ -1023,19 +1020,15 @@ FileStorManagerTest::putDoc(DummyStorageLink& top, document::DocumentId docId(vespalib::make_string("id:ns:testdoctype1:n=%" PRIu64 ":%d", target.getId(), docNum)); document::BucketId bucket(16, factory.getBucketId(docId).getRawId()); //std::cerr << "doc bucket is " << bucket << " vs source " << source << "\n"; - _node->getPersistenceProvider().createBucket( - makeSpiBucket(target), context); + _node->getPersistenceProvider().createBucket(makeSpiBucket(target), context); Document::SP doc(new Document(*_testdoctype1, docId)); - std::shared_ptr<api::PutCommand> cmd( - new api::PutCommand(makeDocumentBucket(target), doc, docNum+1)); + auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(target), doc, docNum+1); cmd->setAddress(address); cmd->setPriority(120); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); - std::shared_ptr<api::PutReply> reply( - std::dynamic_pointer_cast<api::PutReply>( - top.getReply(0))); + std::shared_ptr<api::PutReply> reply(std::dynamic_pointer_cast<api::PutReply>(top.getReply(0))); ASSERT_TRUE(reply.get()); ASSERT_EQ(ReturnCode(ReturnCode::OK), reply->getResult()); top.reset(); @@ -1044,8 +1037,7 @@ FileStorManagerTest::putDoc(DummyStorageLink& top, TEST_F(FileStorManagerTest, split_empty_target_with_remapped_ops) { DummyStorageLink top; DummyStorageLink *dummyManager; - top.push_back(std::unique_ptr<StorageLink>( - dummyManager = new DummyStorageLink)); + top.push_back(std::unique_ptr<StorageLink>(dummyManager = new DummyStorageLink)); setClusterState("storage:2 distributor:1"); top.open(); ForwardingMessageSender messageSender(*dummyManager); @@ -1055,7 +1047,7 @@ TEST_F(FileStorManagerTest, split_empty_target_with_remapped_ops) { FileStorHandler filestorHandler(messageSender, metrics, _node->getComponentRegister()); std::unique_ptr<DiskThread> thread(createThread( *config, *_node, _node->getPersistenceProvider(), - filestorHandler, *metrics.disks[0]->threads[0], 0)); + filestorHandler, *metrics.disks[0]->threads[0])); document::BucketId source(16, 0x10001); @@ -1084,8 +1076,8 @@ TEST_F(FileStorManagerTest, split_empty_target_with_remapped_ops) { putCmd->setAddress(address); putCmd->setPriority(120); - filestorHandler.schedule(splitCmd, 0); - filestorHandler.schedule(putCmd, 0); + filestorHandler.schedule(splitCmd); + filestorHandler.schedule(putCmd); resumeGuard.reset(); // Unpause filestorHandler.flush(true); @@ -1120,7 +1112,7 @@ TEST_F(FileStorManagerTest, notify_on_split_source_ownership_changed) { FileStorHandler filestorHandler(messageSender, metrics, _node->getComponentRegister()); std::unique_ptr<DiskThread> thread(createThread( *config, *_node, _node->getPersistenceProvider(), - filestorHandler, *metrics.disks[0]->threads[0], 0)); + filestorHandler, *metrics.disks[0]->threads[0])); document::BucketId source(getFirstBucketNotOwnedByDistributor(0)); createBucket(source, 0); @@ -1132,7 +1124,7 @@ TEST_F(FileStorManagerTest, notify_on_split_source_ownership_changed) { splitCmd->setPriority(120); splitCmd->setSourceIndex(0); // Source not owned by this distributor. - filestorHandler.schedule(splitCmd, 0); + filestorHandler.schedule(splitCmd); filestorHandler.flush(true); top.waitForMessages(4, _waitTime); // 3 notify cmds + split reply @@ -1161,7 +1153,7 @@ TEST_F(FileStorManagerTest, join) { FileStorHandler filestorHandler(messageSender, metrics, _node->getComponentRegister()); std::unique_ptr<DiskThread> thread(createThread( *config, *_node, _node->getPersistenceProvider(), - filestorHandler, *metrics.disks[0]->threads[0], 0)); + filestorHandler, *metrics.disks[0]->threads[0])); // Creating documents to test with. Different gids, 2 locations. std::vector<document::Document::SP > documents; for (uint32_t i=0; i<20; ++i) { @@ -1182,7 +1174,7 @@ TEST_F(FileStorManagerTest, join) { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), documents[i], 100 + i); auto address = std::make_unique<api::StorageMessageAddress>("storage", lib::NodeType::STORAGE, 3); cmd->setAddress(*address); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto reply = std::dynamic_pointer_cast<api::PutReply>(top.getReply(0)); @@ -1194,7 +1186,7 @@ TEST_F(FileStorManagerTest, join) { auto rcmd = std::make_shared<api::RemoveCommand>( makeDocumentBucket(bucket), documents[i]->getId(), 1000000 + 100 + i); rcmd->setAddress(*address); - filestorHandler.schedule(rcmd, 0); + filestorHandler.schedule(rcmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto rreply = std::dynamic_pointer_cast<api::RemoveReply>(top.getReply(0)); @@ -1209,7 +1201,7 @@ TEST_F(FileStorManagerTest, join) { auto cmd = std::make_shared<api::JoinBucketsCommand>(makeDocumentBucket(document::BucketId(16, 1))); cmd->getSourceBuckets().emplace_back(document::BucketId(17, 0x00001)); cmd->getSourceBuckets().emplace_back(document::BucketId(17, 0x10001)); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto reply = std::dynamic_pointer_cast<api::JoinBucketsReply>(top.getReply(0)); @@ -1224,7 +1216,7 @@ TEST_F(FileStorManagerTest, join) { makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); cmd->setAddress(address); - filestorHandler.schedule(cmd, 0); + filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); auto reply = std::dynamic_pointer_cast<api::GetReply>(top.getReply(0)); diff --git a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp index 1660fed9e38..732cc402641 100644 --- a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp +++ b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp @@ -41,8 +41,7 @@ void ModifiedBucketCheckerTest::SetUp() { _config.reset(new vdstestlib::DirConfig(getStandardConfig(true))); - _node.reset(new TestServiceLayerApp(DiskCount(1), NodeIndex(0), - _config->getConfigId())); + _node.reset(new TestServiceLayerApp(NodeIndex(0), _config->getConfigId())); _node->setupDummyPersistence(); _top.reset(new DummyStorageLink); diff --git a/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp b/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp index 787a63a618c..8c9ea11f1c8 100644 --- a/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp +++ b/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp @@ -33,7 +33,6 @@ TEST_F(SanityCheckedDeleteTest, delete_bucket_fails_when_provider_out_of_sync) { StorBucketDatabase::WrappedEntry entry( _node->getStorageBucketDatabase().get(bucket, "foo", StorBucketDatabase::CREATE_IF_NONEXISTING)); - entry->disk = 0; entry->info = serviceLayerInfo; entry.write(); } diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index df0ea3e6680..565d8b5ee3c 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -176,7 +176,6 @@ MergeHandlerTest::SetUp() { LOG(debug, "Creating %s in bucket database", _bucket.toString().c_str()); bucketdb::StorageBucketInfo bucketDBEntry; - bucketDBEntry.disk = 0; getEnv().getBucketDatabase(_bucket.getBucketSpace()).insert(_bucket.getBucketId(), bucketDBEntry, "mergetestsetup"); LOG(debug, "Creating bucket to merge"); diff --git a/storage/src/tests/persistence/persistencequeuetest.cpp b/storage/src/tests/persistence/persistencequeuetest.cpp index 8edb03b67fa..7e54b45f96a 100644 --- a/storage/src/tests/persistence/persistencequeuetest.cpp +++ b/storage/src/tests/persistence/persistencequeuetest.cpp @@ -34,8 +34,6 @@ public: explicit Fixture(FileStorTestFixture& parent); ~Fixture(); }; - - static constexpr uint16_t _disk = 0; }; PersistenceQueueTest::Fixture::Fixture(FileStorTestFixture& parent_) @@ -58,7 +56,7 @@ PersistenceQueueTest::Fixture::Fixture(FileStorTestFixture& parent_) // and small enough to not slow down testing too much. filestorHandler->setGetNextMessageTimeout(20ms); - stripeId = filestorHandler->getNextStripeId(0); + stripeId = filestorHandler->getNextStripeId(); } PersistenceQueueTest::Fixture::~Fixture() = default; @@ -90,16 +88,16 @@ TEST_F(PersistenceQueueTest, fetch_next_unlocked_message_if_bucket_locked) { // getNextMessage 2 times should then return a lock on the first bucket, // then subsequently on the second, skipping the already locked bucket. // Puts all have same pri, so order is well defined. - f.filestorHandler->schedule(createPut(1234, 0), _disk); - f.filestorHandler->schedule(createPut(1234, 1), _disk); - f.filestorHandler->schedule(createPut(5432, 0), _disk); + f.filestorHandler->schedule(createPut(1234, 0)); + f.filestorHandler->schedule(createPut(1234, 1)); + f.filestorHandler->schedule(createPut(5432, 0)); - auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId); + auto lock0 = f.filestorHandler->getNextMessage(f.stripeId); ASSERT_TRUE(lock0.first.get()); EXPECT_EQ(document::BucketId(16, 1234), dynamic_cast<api::PutCommand&>(*lock0.second).getBucketId()); - auto lock1 = f.filestorHandler->getNextMessage(_disk, f.stripeId); + auto lock1 = f.filestorHandler->getNextMessage(f.stripeId); ASSERT_TRUE(lock1.first.get()); EXPECT_EQ(document::BucketId(16, 5432), dynamic_cast<api::PutCommand&>(*lock1.second).getBucketId()); @@ -108,16 +106,16 @@ TEST_F(PersistenceQueueTest, fetch_next_unlocked_message_if_bucket_locked) { TEST_F(PersistenceQueueTest, shared_locked_operations_allow_concurrent_bucket_access) { Fixture f(*this); - f.filestorHandler->schedule(createGet(1234), _disk); - f.filestorHandler->schedule(createGet(1234), _disk); + f.filestorHandler->schedule(createGet(1234)); + f.filestorHandler->schedule(createGet(1234)); - auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId); + auto lock0 = f.filestorHandler->getNextMessage(f.stripeId); ASSERT_TRUE(lock0.first.get()); EXPECT_EQ(api::LockingRequirements::Shared, lock0.first->lockingRequirements()); // Even though we already have a lock on the bucket, Gets allow shared locking and we // should therefore be able to get another lock. - auto lock1 = f.filestorHandler->getNextMessage(_disk, f.stripeId); + auto lock1 = f.filestorHandler->getNextMessage(f.stripeId); ASSERT_TRUE(lock1.first.get()); EXPECT_EQ(api::LockingRequirements::Shared, lock1.first->lockingRequirements()); } @@ -125,45 +123,45 @@ TEST_F(PersistenceQueueTest, shared_locked_operations_allow_concurrent_bucket_ac TEST_F(PersistenceQueueTest, exclusive_locked_operation_not_started_if_shared_op_active) { Fixture f(*this); - f.filestorHandler->schedule(createGet(1234), _disk); - f.filestorHandler->schedule(createPut(1234, 0), _disk); + f.filestorHandler->schedule(createGet(1234)); + f.filestorHandler->schedule(createPut(1234, 0)); - auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId); + auto lock0 = f.filestorHandler->getNextMessage(f.stripeId); ASSERT_TRUE(lock0.first.get()); EXPECT_EQ(api::LockingRequirements::Shared, lock0.first->lockingRequirements()); // Expected to time out - auto lock1 = f.filestorHandler->getNextMessage(_disk, f.stripeId); + auto lock1 = f.filestorHandler->getNextMessage(f.stripeId); ASSERT_FALSE(lock1.first.get()); } TEST_F(PersistenceQueueTest, shared_locked_operation_not_started_if_exclusive_op_active) { Fixture f(*this); - f.filestorHandler->schedule(createPut(1234, 0), _disk); - f.filestorHandler->schedule(createGet(1234), _disk); + f.filestorHandler->schedule(createPut(1234, 0)); + f.filestorHandler->schedule(createGet(1234)); - auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId); + auto lock0 = f.filestorHandler->getNextMessage(f.stripeId); ASSERT_TRUE(lock0.first.get()); EXPECT_EQ(api::LockingRequirements::Exclusive, lock0.first->lockingRequirements()); // Expected to time out - auto lock1 = f.filestorHandler->getNextMessage(_disk, f.stripeId); + auto lock1 = f.filestorHandler->getNextMessage(f.stripeId); ASSERT_FALSE(lock1.first.get()); } TEST_F(PersistenceQueueTest, exclusive_locked_operation_not_started_if_exclusive_op_active) { Fixture f(*this); - f.filestorHandler->schedule(createPut(1234, 0), _disk); - f.filestorHandler->schedule(createPut(1234, 0), _disk); + f.filestorHandler->schedule(createPut(1234, 0)); + f.filestorHandler->schedule(createPut(1234, 0)); - auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId); + auto lock0 = f.filestorHandler->getNextMessage(f.stripeId); ASSERT_TRUE(lock0.first.get()); EXPECT_EQ(api::LockingRequirements::Exclusive, lock0.first->lockingRequirements()); // Expected to time out - auto lock1 = f.filestorHandler->getNextMessage(_disk, f.stripeId); + auto lock1 = f.filestorHandler->getNextMessage(f.stripeId); ASSERT_FALSE(lock1.first.get()); } diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index 11a876ad0e5..7185271bea4 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -25,13 +25,11 @@ namespace { spi::LoadType defaultLoadType(0, "default"); - vdstestlib::DirConfig initialize(uint32_t numDisks, const std::string & rootOfRoot) { + vdstestlib::DirConfig initialize(const std::string & rootOfRoot) { vdstestlib::DirConfig config(getStandardConfig(true, rootOfRoot)); std::string rootFolder = getRootFolder(config); vespalib::rmdir(rootFolder, true); - for (uint32_t i = 0; i < numDisks; i++) { - vespalib::mkdir(vespalib::make_string("%s/disks/d%d", rootFolder.c_str(), i), true); - } + vespalib::mkdir(vespalib::make_string("%s/disks/d0", rootFolder.c_str()), true); return config; } @@ -47,27 +45,23 @@ namespace { }; } -PersistenceTestEnvironment::PersistenceTestEnvironment(DiskCount numDisks, const std::string & rootOfRoot) - : _config(initialize(numDisks, rootOfRoot)), +PersistenceTestEnvironment::PersistenceTestEnvironment(const std::string & rootOfRoot) + : _config(initialize(rootOfRoot)), _messageKeeper(), - _node(numDisks, NodeIndex(0), _config.getConfigId()), + _node(NodeIndex(0), _config.getConfigId()), _component(_node.getComponentRegister(), "persistence test env"), _metrics(_component.getLoadTypes()->getMetricLoadTypes()) { _node.setupDummyPersistence(); - _metrics.initDiskMetrics(numDisks, _node.getLoadTypes()->getMetricLoadTypes(), 1, 1); - _handler = std::make_unique<FileStorHandler>(_messageKeeper, _metrics, - _node.getComponentRegister()); - for (uint32_t i = 0; i < numDisks; i++) { - _diskEnvs.push_back( - std::make_unique<PersistenceUtil>(_config.getConfigId(), _node.getComponentRegister(), *_handler, - *_metrics.disks[i]->threads[0], i, _node.getPersistenceProvider())); - } + _metrics.initDiskMetrics(1, _node.getLoadTypes()->getMetricLoadTypes(), 1, 1); + _handler = std::make_unique<FileStorHandler>(_messageKeeper, _metrics, _node.getComponentRegister()); + _diskEnv = std::make_unique<PersistenceUtil>(_config.getConfigId(), _node.getComponentRegister(), *_handler, + *_metrics.disks[0]->threads[0], _node.getPersistenceProvider()); } PersistenceTestEnvironment::~PersistenceTestEnvironment() { _handler->close(); - while (!_handler->closed(0)) { + while (!_handler->closed()) { std::this_thread::sleep_for(1ms); } } @@ -76,15 +70,14 @@ PersistenceTestUtils::PersistenceTestUtils() = default; PersistenceTestUtils::~PersistenceTestUtils() = default; std::string -PersistenceTestUtils::dumpBucket(const document::BucketId& bid, uint16_t disk) { - assert(disk == 0u); +PersistenceTestUtils::dumpBucket(const document::BucketId& bid) { return dynamic_cast<spi::dummy::DummyPersistence&>(_env->_node.getPersistenceProvider()).dumpBucket(makeSpiBucket(bid)); } void -PersistenceTestUtils::setupDisks(uint32_t numDisks) { - _env = std::make_unique<PersistenceTestEnvironment>(DiskCount(numDisks), "todo-make-unique-persistencetestutils"); - setupExecutor(numDisks); +PersistenceTestUtils::setupDisks() { + _env = std::make_unique<PersistenceTestEnvironment>("todo-make-unique-persistencetestutils"); + setupExecutor(2); } void @@ -93,27 +86,24 @@ PersistenceTestUtils::setupExecutor(uint32_t numThreads) { } std::unique_ptr<PersistenceThread> -PersistenceTestUtils::createPersistenceThread(uint32_t disk) +PersistenceTestUtils::createPersistenceThread() { return std::make_unique<PersistenceThread>(_sequenceTaskExecutor.get(), _env->_node.getComponentRegister(), _env->_config.getConfigId(),getPersistenceProvider(), - getEnv()._fileStorHandler, getEnv()._metrics, disk); + getEnv()._fileStorHandler, getEnv()._metrics); } document::Document::SP PersistenceTestUtils::schedulePut( uint32_t location, spi::Timestamp timestamp, - uint16_t disk, uint32_t minSize, uint32_t maxSize) { document::Document::SP doc(createRandomDocumentAtLocation( location, timestamp, minSize, maxSize)); - std::shared_ptr<api::StorageMessage> msg( - new api::PutCommand( - makeDocumentBucket(document::BucketId(16, location)), doc, timestamp)); - fsHandler().schedule(msg, disk); + auto msg = std::make_shared<api::PutCommand>(makeDocumentBucket(document::BucketId(16, location)), doc, timestamp); + fsHandler().schedule(msg); return doc; } @@ -150,7 +140,7 @@ PersistenceTestUtils::getBucketStatus(const document::BucketId& id) if (!entry.exist()) { ost << "null"; } else { - ost << entry->getBucketInfo().getDocumentCount() << "," << entry->disk; + ost << entry->getBucketInfo().getDocumentCount(); } return ost.str(); @@ -158,76 +148,52 @@ PersistenceTestUtils::getBucketStatus(const document::BucketId& id) document::Document::SP PersistenceTestUtils::doPutOnDisk( - uint16_t disk, uint32_t location, spi::Timestamp timestamp, uint32_t minSize, uint32_t maxSize) { - document::Document::SP doc(createRandomDocumentAtLocation( - location, timestamp, minSize, maxSize)); - assert(disk == 0u); + document::Document::SP doc(createRandomDocumentAtLocation(location, timestamp, minSize, maxSize)); spi::Bucket b(makeSpiBucket(document::BucketId(16, location))); - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - + spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); getPersistenceProvider().createBucket(b, context); - getPersistenceProvider().put(spi::Bucket(b), timestamp, doc, context); - return doc; } bool PersistenceTestUtils::doRemoveOnDisk( - uint16_t disk, const document::BucketId& bucketId, const document::DocumentId& docId, spi::Timestamp timestamp, bool persistRemove) { - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - assert(disk == 0u); + spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); if (persistRemove) { - spi::RemoveResult result = getPersistenceProvider().removeIfFound( - makeSpiBucket(bucketId), - timestamp, docId, context); + spi::RemoveResult result = getPersistenceProvider().removeIfFound(makeSpiBucket(bucketId),timestamp, docId, context); return result.wasFound(); } - spi::RemoveResult result = getPersistenceProvider().remove( - makeSpiBucket(bucketId), - timestamp, docId, context); + spi::RemoveResult result = getPersistenceProvider().remove(makeSpiBucket(bucketId), timestamp, docId, context); return result.wasFound(); } bool PersistenceTestUtils::doUnrevertableRemoveOnDisk( - uint16_t disk, const document::BucketId& bucketId, const document::DocumentId& docId, spi::Timestamp timestamp) { - assert(disk == 0u); - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - spi::RemoveResult result = getPersistenceProvider().remove( - makeSpiBucket(bucketId), - timestamp, docId, context); + spi::Context context(defaultLoadType, spi::Priority(0),spi::Trace::TraceLevel(0)); + spi::RemoveResult result = getPersistenceProvider().remove(makeSpiBucket(bucketId), timestamp, docId, context); return result.wasFound(); } spi::GetResult -PersistenceTestUtils::doGetOnDisk( - uint16_t disk, - const document::BucketId& bucketId, - const document::DocumentId& docId) +PersistenceTestUtils::doGetOnDisk(const document::BucketId& bucketId, const document::DocumentId& docId) { auto fieldSet = std::make_unique<document::AllFields>(); - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - assert(disk == 0u); + spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); return getPersistenceProvider().get(makeSpiBucket(bucketId), *fieldSet, docId, context); } @@ -255,47 +221,19 @@ PersistenceTestUtils::createHeaderUpdate(const document::DocumentId& docId, cons return update; } -uint16_t -PersistenceTestUtils::getDiskFromBucketDatabaseIfUnset(const document::Bucket& bucket, uint16_t disk) -{ - if (disk == 0xffff) { - StorBucketDatabase::WrappedEntry entry( - getEnv().getBucketDatabase(bucket.getBucketSpace()).get(bucket.getBucketId(), "createTestBucket")); - if (entry.exist()) { - return entry->disk; - } else { - std::ostringstream error; - error << bucket.toString() << " not in db and disk unset"; - throw vespalib::IllegalStateException(error.str(), VESPA_STRLOC); - } - } - return disk; -} - void -PersistenceTestUtils::doPut(const document::Document::SP& doc, - spi::Timestamp time, - uint16_t disk, - uint16_t usedBits) +PersistenceTestUtils::doPut(const document::Document::SP& doc, spi::Timestamp time, uint16_t usedBits) { - document::BucketId bucket( - _env->_component.getBucketIdFactory().getBucketId(doc->getId())); + document::BucketId bucket(_env->_component.getBucketIdFactory().getBucketId(doc->getId())); bucket.setUsedBits(usedBits); - disk = getDiskFromBucketDatabaseIfUnset(makeDocumentBucket(bucket), disk); - - doPut(doc, bucket, time, disk); + doPut(doc, bucket, time); } void -PersistenceTestUtils::doPut(const document::Document::SP& doc, - document::BucketId bid, - spi::Timestamp time, - uint16_t disk) +PersistenceTestUtils::doPut(const document::Document::SP& doc, document::BucketId bid, spi::Timestamp time) { - assert(disk == 0u); spi::Bucket b(makeSpiBucket(bid)); - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); + spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); getPersistenceProvider().createBucket(b, context); getPersistenceProvider().put(b, time, std::move(doc), context); } @@ -303,28 +241,20 @@ PersistenceTestUtils::doPut(const document::Document::SP& doc, spi::UpdateResult PersistenceTestUtils::doUpdate(document::BucketId bid, const document::DocumentUpdate::SP& update, - spi::Timestamp time, - uint16_t disk) + spi::Timestamp time) { - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - assert(disk == 0u); - return getPersistenceProvider().update( - makeSpiBucket(bid), time, update, context); + spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); + return getPersistenceProvider().update(makeSpiBucket(bid), time, update, context); } void PersistenceTestUtils::doRemove(const document::DocumentId& id, spi::Timestamp time, - uint16_t disk, bool unrevertableRemove, - uint16_t usedBits) + bool unrevertableRemove, uint16_t usedBits) { document::BucketId bucket( _env->_component.getBucketIdFactory().getBucketId(id)); bucket.setUsedBits(usedBits); - disk = getDiskFromBucketDatabaseIfUnset(makeDocumentBucket(bucket), disk); - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - assert(disk == 0u); + spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); if (unrevertableRemove) { getPersistenceProvider().remove( makeSpiBucket(bucket), time, id, context); @@ -360,8 +290,7 @@ PersistenceTestUtils::createRandomDocumentAtLocation( } void -PersistenceTestUtils::createTestBucket(const document::Bucket& bucket, - uint16_t disk) +PersistenceTestUtils::createTestBucket(const document::Bucket& bucket) { document::BucketId bucketId(bucket.getBucketId()); uint32_t opsPerType = 2; @@ -377,25 +306,20 @@ PersistenceTestUtils::createTestBucket(const document::Bucket& bucket, location <<= 32; location += (bucketId.getRawId() & 0xffffffff); document::Document::SP doc( - createRandomDocumentAtLocation( - location, seed, minDocSize, maxDocSize)); + createRandomDocumentAtLocation(location, seed, minDocSize, maxDocSize)); if (headerOnly) { clearBody(*doc); } - doPut(doc, spi::Timestamp(seed), disk, bucketId.getUsedBits()); + doPut(doc, spi::Timestamp(seed), bucketId.getUsedBits()); if (optype == 0) { // Regular put } else if (optype == 1) { // Overwritten later in time document::Document::SP doc2(new document::Document(*doc)); - doc2->setValue(doc2->getField("content"), - document::StringFieldValue("overwritten")); - doPut(doc2, spi::Timestamp(seed + 500), - disk, bucketId.getUsedBits()); + doc2->setValue(doc2->getField("content"), document::StringFieldValue("overwritten")); + doPut(doc2, spi::Timestamp(seed + 500), bucketId.getUsedBits()); } else if (optype == 2) { // Removed - doRemove(doc->getId(), spi::Timestamp(seed + 500), disk, false, - bucketId.getUsedBits()); + doRemove(doc->getId(), spi::Timestamp(seed + 500), false, bucketId.getUsedBits()); } else if (optype == 3) { // Unrevertable removed - doRemove(doc->getId(), spi::Timestamp(seed), disk, true, - bucketId.getUsedBits()); + doRemove(doc->getId(), spi::Timestamp(seed), true, bucketId.getUsedBits()); } } } diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h index d889deabbd5..a0ab516754a 100644 --- a/storage/src/tests/persistence/persistencetestutils.h +++ b/storage/src/tests/persistence/persistencetestutils.h @@ -24,7 +24,7 @@ struct MessageKeeper : public MessageSender { }; struct PersistenceTestEnvironment { - PersistenceTestEnvironment(DiskCount numDisks, const std::string & rootOfRoot); + PersistenceTestEnvironment(const std::string & rootOfRoot); ~PersistenceTestEnvironment(); document::TestDocMan _testDocMan; @@ -34,7 +34,7 @@ struct PersistenceTestEnvironment { StorageComponent _component; FileStorMetrics _metrics; std::unique_ptr<FileStorHandler> _handler; - std::vector<std::unique_ptr<PersistenceUtil> > _diskEnvs; + std::unique_ptr<PersistenceUtil> _diskEnv; }; class PersistenceTestUtils : public testing::Test { @@ -76,14 +76,9 @@ public: PersistenceTestUtils(); virtual ~PersistenceTestUtils(); - document::Document::SP schedulePut( - uint32_t location, - spi::Timestamp timestamp, - uint16_t disk, - uint32_t minSize = 0, - uint32_t maxSize = 128); + document::Document::SP schedulePut(uint32_t location, spi::Timestamp timestamp, uint32_t minSize = 0, uint32_t maxSize = 128); - void setupDisks(uint32_t disks); + void setupDisks(); void setupExecutor(uint32_t numThreads); void TearDown() override { @@ -94,10 +89,9 @@ public: _env.reset(); } - std::string dumpBucket(const document::BucketId& bid, uint16_t disk = 0); + std::string dumpBucket(const document::BucketId& bid); - PersistenceUtil& getEnv(uint32_t disk = 0) - { return *_env->_diskEnvs[disk]; } + PersistenceUtil& getEnv() { return *_env->_diskEnv; } FileStorHandler& fsHandler() { return *_env->_handler; } FileStorMetrics& metrics() { return _env->_metrics; } MessageKeeper& messageKeeper() { return _env->_messageKeeper; } @@ -128,11 +122,9 @@ public: } /** - Performs a put to the given disk. Returns the document that was inserted. */ document::Document::SP doPutOnDisk( - uint16_t disk, uint32_t location, spi::Timestamp timestamp, uint32_t minSize = 0, @@ -143,14 +135,12 @@ public: spi::Timestamp timestamp, uint32_t minSize = 0, uint32_t maxSize = 128) - { return doPutOnDisk(0, location, timestamp, minSize, maxSize); } + { return doPutOnDisk(location, timestamp, minSize, maxSize); } /** - Performs a remove to the given disk. Returns the new doccount if document was removed, or -1 if not found. */ bool doRemoveOnDisk( - uint16_t disk, const document::BucketId& bid, const document::DocumentId& id, spi::Timestamp timestamp, @@ -161,11 +151,10 @@ public: const document::DocumentId& id, spi::Timestamp timestamp, bool persistRemove) { - return doRemoveOnDisk(0, bid, id, timestamp, persistRemove); + return doRemoveOnDisk(bid, id, timestamp, persistRemove); } - bool doUnrevertableRemoveOnDisk(uint16_t disk, - const document::BucketId& bid, + bool doUnrevertableRemoveOnDisk(const document::BucketId& bid, const document::DocumentId& id, spi::Timestamp timestamp); @@ -173,29 +162,27 @@ public: const document::DocumentId& id, spi::Timestamp timestamp) { - return doUnrevertableRemoveOnDisk(0, bid, id, timestamp); + return doUnrevertableRemoveOnDisk(bid, id, timestamp); } /** * Do a remove toward storage set up in test environment. * * @id Document to remove. - * @disk If set, use this disk, otherwise lookup in bucket db. * @unrevertableRemove If set, instead of adding put, turn put to remove. * @usedBits Generate bucket to use from docid using this amount of bits. */ - void doRemove(const document::DocumentId& id, spi::Timestamp, uint16_t disk = 0xffff, + void doRemove(const document::DocumentId& id, spi::Timestamp, bool unrevertableRemove = false, uint16_t usedBits = 16); spi::GetResult doGetOnDisk( - uint16_t disk, const document::BucketId& bucketId, const document::DocumentId& docId); spi::GetResult doGet( const document::BucketId& bucketId, const document::DocumentId& docId) - { return doGetOnDisk(0, bucketId, docId); } + { return doGetOnDisk(bucketId, docId); } std::shared_ptr<document::DocumentUpdate> createBodyUpdate( const document::DocumentId& id, @@ -205,28 +192,23 @@ public: const document::DocumentId& id, const document::FieldValue& updateValue); - uint16_t getDiskFromBucketDatabaseIfUnset(const document::Bucket &, - uint16_t disk = 0xffff); + uint16_t getDiskFromBucketDatabaseIfUnset(const document::Bucket &); /** * Do a put toward storage set up in test environment. * * @doc Document to put. Use TestDocMan to generate easily. - * @disk If set, use this disk, otherwise lookup in bucket db. * @usedBits Generate bucket to use from docid using this amount of bits. */ - void doPut(const document::Document::SP& doc, spi::Timestamp, - uint16_t disk = 0xffff, uint16_t usedBits = 16); + void doPut(const document::Document::SP& doc, spi::Timestamp, uint16_t usedBits = 16); void doPut(const document::Document::SP& doc, document::BucketId bid, - spi::Timestamp time, - uint16_t disk = 0); + spi::Timestamp time); spi::UpdateResult doUpdate(document::BucketId bid, const std::shared_ptr<document::DocumentUpdate>& update, - spi::Timestamp time, - uint16_t disk = 0); + spi::Timestamp time); document::Document::UP createRandomDocumentAtLocation( uint64_t location, uint32_t seed, @@ -237,14 +219,13 @@ public: * bucket can represent. (Such that tests have a nice test bucket to use * that require operations to handle all the various bucket contents. * - * @disk If set, use this disk, otherwise lookup in bucket db. */ - void createTestBucket(const document::Bucket&, uint16_t disk = 0xffff); + void createTestBucket(const document::Bucket&); /** * Create a new persistence thread. */ - std::unique_ptr<PersistenceThread> createPersistenceThread(uint32_t disk); + std::unique_ptr<PersistenceThread> createPersistenceThread(); /** * In-place modify doc so that it has no more body fields. @@ -256,7 +237,7 @@ class SingleDiskPersistenceTestUtils : public PersistenceTestUtils { public: void SetUp() override { - setupDisks(1); + setupDisks(); } }; diff --git a/storage/src/tests/persistence/persistencethread_splittest.cpp b/storage/src/tests/persistence/persistencethread_splittest.cpp index 3d7fc70db6a..10fb0b2e6b4 100644 --- a/storage/src/tests/persistence/persistencethread_splittest.cpp +++ b/storage/src/tests/persistence/persistencethread_splittest.cpp @@ -204,7 +204,7 @@ PersistenceThreadSplitTest::doTest(SplitCase splitCase) spi.put(bucket, spi::Timestamp(1000 + i), std::move(doc), context); } - std::unique_ptr<PersistenceThread> thread(createPersistenceThread(0)); + std::unique_ptr<PersistenceThread> thread(createPersistenceThread()); getNode().getStateUpdater().setClusterState( std::make_shared<lib::ClusterState>("distributor:1 storage:1")); document::Bucket docBucket = makeDocumentBucket(document::BucketId(currentSplitLevel, 1)); diff --git a/storage/src/tests/persistence/processalltest.cpp b/storage/src/tests/persistence/processalltest.cpp index 5174b733334..24c67fe93c1 100644 --- a/storage/src/tests/persistence/processalltest.cpp +++ b/storage/src/tests/persistence/processalltest.cpp @@ -42,7 +42,7 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { for (int i = 0; i < 10; ++i) { document::Document::SP doc(docMan.createRandomDocumentAtLocation(4, 1234 + i)); doc->setValue(doc->getField("headerval"), document::IntFieldValue(i)); - doPut(doc, bucketId, spi::Timestamp(100 + i), 0); + doPut(doc, bucketId, spi::Timestamp(100 + i)); } document::Bucket bucket = makeDocumentBucket(bucketId); @@ -102,7 +102,7 @@ TEST_F(ProcessAllHandlerTest, bucket_stat_request_returns_document_metadata_matc for (int i = 0; i < 10; ++i) { document::Document::SP doc(docMan.createRandomDocumentAtLocation(4, 1234 + i)); doc->setValue(doc->getField("headerval"), document::IntFieldValue(i)); - doPut(doc, bucketId, spi::Timestamp(100 + i), 0); + doPut(doc, bucketId, spi::Timestamp(100 + i)); } document::Bucket bucket = makeDocumentBucket(bucketId); @@ -114,7 +114,7 @@ TEST_F(ProcessAllHandlerTest, bucket_stat_request_returns_document_metadata_matc EXPECT_EQ(api::ReturnCode::OK, reply.getResult().getResult()); vespalib::string expected = - "Persistence bucket BucketId(0x4000000000000004), partition 0\n" + "Persistence bucket BucketId(0x4000000000000004)\n" " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 163\n" " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 141\n" " Timestamp: 104, Doc(id:mail:testdoctype1:n=4:56061.html), gid(0x040000002b8f80f0160f6c5c), size: 118\n" @@ -132,7 +132,7 @@ TEST_F(ProcessAllHandlerTest, stat_bucket_request_can_returned_removed_entries) for (int i = 0; i < 10; ++i) { document::Document::SP doc(docMan.createRandomDocumentAtLocation(4, 1234 + i)); doc->setValue(doc->getField("headerval"), document::IntFieldValue(i)); - doPut(doc, bucketId, spi::Timestamp(100 + i), 0); + doPut(doc, bucketId, spi::Timestamp(100 + i)); doRemove(bucketId, doc->getId(), spi::Timestamp(200 + i), @@ -148,7 +148,7 @@ TEST_F(ProcessAllHandlerTest, stat_bucket_request_can_returned_removed_entries) EXPECT_EQ(api::ReturnCode::OK, reply.getResult().getResult()); vespalib::string expected = - "Persistence bucket BucketId(0x4000000000000004), partition 0\n" + "Persistence bucket BucketId(0x4000000000000004)\n" " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 163\n" " Timestamp: 101, Doc(id:mail:testdoctype1:n=4:33113.html), gid(0x04000000b121a632741db368), size: 89\n" " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 141\n" @@ -182,7 +182,7 @@ TEST_F(ProcessAllHandlerTest, bucket_stat_request_can_return_all_put_entries_in_ for (int i = 0; i < 10; ++i) { document::Document::SP doc(docMan.createRandomDocumentAtLocation(4, 1234 + i)); doc->setValue(doc->getField("headerval"), document::IntFieldValue(i)); - doPut(doc, bucketId, spi::Timestamp(100 + i), 0); + doPut(doc, bucketId, spi::Timestamp(100 + i)); } document::Bucket bucket = makeDocumentBucket(bucketId); @@ -194,7 +194,7 @@ TEST_F(ProcessAllHandlerTest, bucket_stat_request_can_return_all_put_entries_in_ EXPECT_EQ(api::ReturnCode::OK, reply.getResult().getResult()); vespalib::string expected = - "Persistence bucket BucketId(0x4000000000000004), partition 0\n" + "Persistence bucket BucketId(0x4000000000000004)\n" " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 163\n" " Timestamp: 101, Doc(id:mail:testdoctype1:n=4:33113.html), gid(0x04000000b121a632741db368), size: 89\n" " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 141\n" diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index 18d78e1e8cf..ce041660a2f 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -47,7 +47,7 @@ struct TestAndSetTest : SingleDiskPersistenceTestUtils { createBucket(BUCKET_ID); getPersistenceProvider().createBucket(makeSpiBucket(BUCKET_ID),context); - thread = createPersistenceThread(0); + thread = createPersistenceThread(); testDoc = createTestDocument(); testDocId = testDoc->getId(); } |