diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-05-05 00:23:55 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-05-05 00:23:55 +0200 |
commit | 8bf5ae859e0664c8fd797243328baf6dc1717f7e (patch) | |
tree | c89271841f20d46e62c4973aab87fb6ce320d197 | |
parent | df7cce5d7d2ee808ed31fa98029b38117f39703d (diff) | |
parent | 5ae2f92fc1a7a32d6bb3b60e7debd8180d75e80c (diff) |
Merge pull request #13150 from vespa-engine/vekterli/remove-deprecated-bucket-disk-move-functionality
Remove deprecated bucket cross-disk move functionality
35 files changed, 15 insertions, 1919 deletions
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 52423f031e5..e9f2d736b91 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -24,7 +24,6 @@ vespa_define_module( LIBS src/vespa/storage src/vespa/storage/bucketdb - src/vespa/storage/bucketmover src/vespa/storage/common src/vespa/storage/common/hostreporter src/vespa/storage/config @@ -52,7 +51,6 @@ vespa_define_module( TESTS src/tests src/tests/bucketdb - src/tests/bucketmover src/tests/common src/tests/common/hostreporter src/tests/distributor diff --git a/storage/src/tests/bucketmover/.gitignore b/storage/src/tests/bucketmover/.gitignore deleted file mode 100644 index 2f86228e663..00000000000 --- a/storage/src/tests/bucketmover/.gitignore +++ /dev/null @@ -1 +0,0 @@ -storage_bucketmover_gtest_runner_app diff --git a/storage/src/tests/bucketmover/CMakeLists.txt b/storage/src/tests/bucketmover/CMakeLists.txt deleted file mode 100644 index c7a6e8fc21e..00000000000 --- a/storage/src/tests/bucketmover/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(storage_bucketmover_gtest_runner_app TEST - SOURCES - bucketmovertest.cpp - htmltabletest.cpp - gtest_runner.cpp - DEPENDS - storage - storage_testcommon - gtest -) - -vespa_add_test( - NAME storage_bucketmover_gtest_runner_app - COMMAND storage_bucketmover_gtest_runner_app -) diff --git a/storage/src/tests/bucketmover/bucketmovertest.cpp b/storage/src/tests/bucketmover/bucketmovertest.cpp deleted file mode 100644 index e360819c5d7..00000000000 --- a/storage/src/tests/bucketmover/bucketmovertest.cpp +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/storage/bucketdb/storbucketdb.h> -#include <vespa/storage/common/bucketmessages.h> -#include <vespa/storage/bucketmover/bucketmover.h> -#include <vespa/document/test/make_bucket_space.h> -#include <vespa/document/test/make_document_bucket.h> -#include <tests/common/dummystoragelink.h> -#include <tests/common/teststorageapp.h> -#include <vespa/config/common/exceptions.h> -#include <gtest/gtest.h> - -bool debug = false; - -using document::test::makeBucketSpace; -using document::test::makeDocumentBucket; - -namespace storage::bucketmover { - -struct BucketMoverTest : public ::testing::Test { -public: - void SetUp() override; - void TearDown() override; - - std::unique_ptr<TestServiceLayerApp> _node; - std::unique_ptr<ServiceLayerComponent> _component; - std::unique_ptr<BucketMover> _bucketMover; - DummyStorageLink* after; - - void addBucket(const document::BucketId& id, uint16_t idealDiff); -}; - -void -BucketMoverTest::TearDown() -{ - _node.reset(0); -} - -void -BucketMoverTest::SetUp() -{ - try { - _node.reset(new TestServiceLayerApp(DiskCount(4))); - _node->setupDummyPersistence(); - } catch (config::InvalidConfigException& e) { - fprintf(stderr, "%s\n", e.what()); - } - - _component.reset(new ServiceLayerComponent(_node->getComponentRegister(), "foo")); - _bucketMover.reset(new BucketMover("raw:", _node->getComponentRegister())); - after = new DummyStorageLink(); - _bucketMover->push_back(StorageLink::UP(after)); -} - -void -BucketMoverTest::addBucket(const document::BucketId& id, - uint16_t idealDiff) -{ - StorBucketDatabase::WrappedEntry entry( - _component->getBucketDatabase(makeBucketSpace()).get( - id, - "", - StorBucketDatabase::CREATE_IF_NONEXISTING)); - - entry->setBucketInfo(api::BucketInfo(1,1,1)); - - uint16_t idealDisk = _component->getIdealPartition(makeDocumentBucket(id)); - entry->disk = (idealDisk + idealDiff) % _component->getDiskCount(); - entry.write(); -} - -TEST_F(BucketMoverTest, testNormalUsage) -{ - for (uint32_t i = 1; i < 4; ++i) { - addBucket(document::BucketId(16, i), 1); - } - for (uint32_t i = 4; i < 6; ++i) { - addBucket(document::BucketId(16, i), 0); - } - - _bucketMover->open(); - _bucketMover->tick(); - - std::vector<api::StorageMessage::SP> msgs = after->getCommandsOnce(); - EXPECT_EQ( - std::string("BucketDiskMoveCommand(" - "BucketId(0x4000000000000002), source 3, target 2)"), - msgs[0]->toString()); - EXPECT_EQ( - std::string("BucketDiskMoveCommand(" - "BucketId(0x4000000000000001), source 2, target 1)"), - msgs[1]->toString()); - EXPECT_EQ( - std::string("BucketDiskMoveCommand(" - "BucketId(0x4000000000000003), source 1, target 0)"), - msgs[2]->toString()); - - for (uint32_t i = 0; i < 2; ++i) { - after->sendUp(std::shared_ptr<api::StorageMessage>( - ((api::StorageCommand*)msgs[i].get())-> - makeReply().release())); - } - - _bucketMover->tick(); - EXPECT_EQ(0, (int)after->getNumCommands()); - - _bucketMover->finishCurrentRun(); -} - -TEST_F(BucketMoverTest, testMaxPending) -{ - for (uint32_t i = 1; i < 100; ++i) { - addBucket(document::BucketId(16, i), 1); - } - for (uint32_t i = 101; i < 200; ++i) { - addBucket(document::BucketId(16, i), 0); - } - - _bucketMover->open(); - _bucketMover->tick(); - - std::vector<api::StorageMessage::SP> msgs = after->getCommandsOnce(); - // 5 is the max pending default config. - ASSERT_EQ(5, (int)msgs.size()); - - after->sendUp(std::shared_ptr<api::StorageMessage>( - ((api::StorageCommand*)msgs[3].get())-> - makeReply().release())); - - _bucketMover->tick(); - - std::vector<api::StorageMessage::SP> msgs2 = after->getCommandsOnce(); - ASSERT_EQ(1, (int)msgs2.size()); -} - -TEST_F(BucketMoverTest, testErrorHandling) -{ - for (uint32_t i = 1; i < 100; ++i) { - addBucket(document::BucketId(16, i), 1); - } - for (uint32_t i = 101; i < 200; ++i) { - addBucket(document::BucketId(16, i), 0); - } - - _bucketMover->open(); - _bucketMover->tick(); - - std::vector<api::StorageMessage::SP> msgs = after->getCommandsOnce(); - // 5 is the max pending default config. - ASSERT_EQ(5, (int)msgs.size()); - - BucketDiskMoveCommand& cmd = static_cast<BucketDiskMoveCommand&>(*msgs[0]); - uint32_t targetDisk = cmd.getDstDisk(); - - std::unique_ptr<api::StorageReply> reply(cmd.makeReply().release()); - reply->setResult(api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE, "foobar")); - after->sendUp(std::shared_ptr<api::StorageMessage>(reply.release())); - - for (uint32_t i = 1; i < msgs.size(); ++i) { - after->sendUp(std::shared_ptr<api::StorageMessage>( - ((api::StorageCommand*)msgs[i].get())-> - makeReply().release())); - } - - _bucketMover->tick(); - - std::vector<api::StorageMessage::SP> msgs2 = after->getCommandsOnce(); - ASSERT_EQ(5, (int)msgs2.size()); - - for (uint32_t i = 0; i < msgs2.size(); ++i) { - BucketDiskMoveCommand& bdm = static_cast<BucketDiskMoveCommand&>(*msgs2[i]); - EXPECT_NE(bdm.getDstDisk(), targetDisk); - } -} - -} diff --git a/storage/src/tests/bucketmover/gtest_runner.cpp b/storage/src/tests/bucketmover/gtest_runner.cpp deleted file mode 100644 index 3d49ada55a7..00000000000 --- a/storage/src/tests/bucketmover/gtest_runner.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/gtest/gtest.h> - -#include <vespa/log/log.h> -LOG_SETUP("storage_bucketmover_gtest_runner"); - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/storage/src/tests/frameworkimpl/status/CMakeLists.txt b/storage/src/tests/frameworkimpl/status/CMakeLists.txt index 655dff1bd0c..1b49b1bac45 100644 --- a/storage/src/tests/frameworkimpl/status/CMakeLists.txt +++ b/storage/src/tests/frameworkimpl/status/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(storage_status_gtest_runner_app TEST SOURCES gtest_runner.cpp + htmltabletest.cpp statustest.cpp DEPENDS storage diff --git a/storage/src/tests/bucketmover/htmltabletest.cpp b/storage/src/tests/frameworkimpl/status/htmltabletest.cpp index 8791ae1ec8d..132dee55d93 100644 --- a/storage/src/tests/bucketmover/htmltabletest.cpp +++ b/storage/src/tests/frameworkimpl/status/htmltabletest.cpp @@ -1,13 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/bucketmover/htmltable.h> +#include <vespa/storage/frameworkimpl/thread/htmltable.h> #include <gtest/gtest.h> namespace storage { TEST(HtmlTableTest, testPercentageColumn) { - // With total hardcoded to 100 + // With total hardcoded to 100 { HtmlTable table("disk"); PercentageColumn perc("fillrate", 100); @@ -24,15 +24,15 @@ TEST(HtmlTableTest, testPercentageColumn) std::ostringstream ost; table.print(ost); std::string expected( -"<table border=\"1\" cellpadding=\"2\" cellspacing=\"0\">\n" -"<tr><th>disk</th><th>fillrate</th></tr>\n" -"<tr><td>0</td><td bgcolor=\"#a0ffa0\" align=\"right\">30.00 %</td></tr>\n" -"<tr><td>1</td><td bgcolor=\"#ffffa0\" align=\"right\">80.00 %</td></tr>\n" -"<tr><td>2</td><td bgcolor=\"#ffa0a0\" align=\"right\">100.00 %</td></tr>\n" -"</table>\n"); + "<table border=\"1\" cellpadding=\"2\" cellspacing=\"0\">\n" + "<tr><th>disk</th><th>fillrate</th></tr>\n" + "<tr><td>0</td><td bgcolor=\"#a0ffa0\" align=\"right\">30.00 %</td></tr>\n" + "<tr><td>1</td><td bgcolor=\"#ffffa0\" align=\"right\">80.00 %</td></tr>\n" + "<tr><td>2</td><td bgcolor=\"#ffa0a0\" align=\"right\">100.00 %</td></tr>\n" + "</table>\n"); EXPECT_EQ(expected, ost.str()); } - // With automatically gathered total + // With automatically gathered total { HtmlTable table("disk"); PercentageColumn perc("fillrate"); @@ -65,7 +65,7 @@ TEST(HtmlTableTest, testByteSizeColumn) table.addRow(0); table.addRow(1); table.addRow(2); - // Biggest value enforce the denomination + // Biggest value enforce the denomination size[0] = 42123; size[1] = 124123151; size[2] = 6131231; @@ -83,4 +83,4 @@ TEST(HtmlTableTest, testByteSizeColumn) } -} // storage +} // storage
\ No newline at end of file diff --git a/storage/src/tests/persistence/CMakeLists.txt b/storage/src/tests/persistence/CMakeLists.txt index 1e25f38fe2a..e8095109806 100644 --- a/storage/src/tests/persistence/CMakeLists.txt +++ b/storage/src/tests/persistence/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_executable(storage_persistence_gtest_runner_app TEST SOURCES bucketownershipnotifiertest.cpp - diskmoveoperationhandlertest.cpp mergehandlertest.cpp persistencequeuetest.cpp persistencetestutils.cpp diff --git a/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp b/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp deleted file mode 100644 index c10681405e7..00000000000 --- a/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/storage/persistence/diskmoveoperationhandler.h> -#include <vespa/storage/persistence/messages.h> -#include <tests/persistence/persistencetestutils.h> -#include <vespa/document/test/make_document_bucket.h> - -using document::test::makeDocumentBucket; -using namespace ::testing; - -namespace storage { - -struct DiskMoveOperationHandlerTest : PersistenceTestUtils {}; - -TEST_F(DiskMoveOperationHandlerTest, simple) { - setupDisks(10); - - // Create bucket 16, 4 on disk 3. - { - StorBucketDatabase::WrappedEntry entry( - createBucket(document::BucketId(16, 4))); - entry->disk = 3; - entry.write(); - } - - for (uint32_t i = 0; i < 10; i++) { - doPutOnDisk(3, 4, spi::Timestamp(1000 + i)); - } - - DiskMoveOperationHandler diskMoveHandler(getEnv(3),getPersistenceProvider()); - document::Bucket bucket = makeDocumentBucket(document::BucketId(16, 4)); - auto move = std::make_shared<BucketDiskMoveCommand>(bucket, 3, 4); - spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); - diskMoveHandler.handleBucketDiskMove(*move, std::make_unique<MessageTracker>(getEnv(), NoBucketLock::make(bucket), move)); - - EXPECT_EQ("BucketId(0x4000000000000004): 10,4", - getBucketStatus(document::BucketId(16,4))); -} - -} diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 6497f8bb698..fe966d0bbb2 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -338,76 +338,6 @@ TEST_F(FileStorManagerTest, put) { } } -TEST_F(FileStorManagerTest, disk_move) { - setupDisks(2); - - // Setting up manager - DummyStorageLink top; - FileStorManager *manager; - top.push_back(unique_ptr<StorageLink>(manager = - new FileStorManager(config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); - top.open(); - api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); - // Creating a document to test with - Document::SP doc(createDocument( - "some content", "id:crawler:testdoctype1:n=4000:foo").release()); - - document::BucketId bid(16, 4000); - - createBucket(bid, 0); - - // Putting it - { - auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(address); - top.sendDown(cmd); - top.waitForMessages(1, _waitTime); - ASSERT_EQ(1, top.getNumReplies()); - auto reply = std::dynamic_pointer_cast<api::PutReply>(top.getReply(0)); - top.reset(); - ASSERT_TRUE(reply.get()); - EXPECT_EQ(ReturnCode(ReturnCode::OK), reply->getResult()); - EXPECT_EQ(1, reply->getBucketInfo().getDocumentCount()); - } - - { - StorBucketDatabase::WrappedEntry entry( - _node->getStorageBucketDatabase().get(bid, "foo")); - - EXPECT_EQ(0, entry->disk); - EXPECT_EQ( - vespalib::string( - "BucketInfo(crc 0x3538028e, docCount 1, totDocSize 124, " - "ready true, active false)"), - entry->getBucketInfo().toString()); - } - - { - auto cmd = std::make_shared<BucketDiskMoveCommand>(makeDocumentBucket(bid), 0, 1); - - top.sendDown(cmd); - top.waitForMessages(1, _waitTime); - ASSERT_EQ(1, top.getNumReplies()); - auto reply = std::dynamic_pointer_cast<BucketDiskMoveReply>(top.getReply(0)); - top.reset(); - ASSERT_TRUE(reply.get()); - EXPECT_EQ(ReturnCode(ReturnCode::OK), reply->getResult()); - EXPECT_EQ(1, reply->getBucketInfo().getDocumentCount()); - } - - { - StorBucketDatabase::WrappedEntry entry( - _node->getStorageBucketDatabase().get(bid, "foo")); - - EXPECT_EQ(1, entry->disk); - EXPECT_EQ( - vespalib::string( - "BucketInfo(crc 0x3538028e, docCount 1, totDocSize 124, " - "ready true, active false)"), - entry->getBucketInfo().toString()); - } -} - TEST_F(FileStorManagerTest, state_change) { // Setting up manager DummyStorageLink top; diff --git a/storage/src/vespa/storage/CMakeLists.txt b/storage/src/vespa/storage/CMakeLists.txt index d854c603097..100118b5133 100644 --- a/storage/src/vespa/storage/CMakeLists.txt +++ b/storage/src/vespa/storage/CMakeLists.txt @@ -9,7 +9,6 @@ vespa_add_library(storage $<TARGET_OBJECTS:storage_spersistence> $<TARGET_OBJECTS:storage_storageutil> $<TARGET_OBJECTS:storage_visitor> - $<TARGET_OBJECTS:storage_bucketmover> $<TARGET_OBJECTS:storage_thread> $<TARGET_OBJECTS:storage_status> $<TARGET_OBJECTS:storage_component> diff --git a/storage/src/vespa/storage/bucketmover/CMakeLists.txt b/storage/src/vespa/storage/bucketmover/CMakeLists.txt deleted file mode 100644 index 669c13d3bc1..00000000000 --- a/storage/src/vespa/storage/bucketmover/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_bucketmover OBJECT - SOURCES - bucketmover.cpp - htmltable.cpp - move.cpp - run.cpp - runstatistics.cpp - DEPENDS - AFTER - storage_storageconfig -) diff --git a/storage/src/vespa/storage/bucketmover/bucketmover.cpp b/storage/src/vespa/storage/bucketmover/bucketmover.cpp deleted file mode 100644 index 904e8a66c27..00000000000 --- a/storage/src/vespa/storage/bucketmover/bucketmover.cpp +++ /dev/null @@ -1,528 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bucketmover.h" -#include "htmltable.h" -#include <vespa/storage/config/config-stor-server.h> -#include <vespa/storage/common/bucketmessages.h> -#include <vespa/storage/common/content_bucket_space_repo.h> -#include <vespa/storage/common/nodestateupdater.h> -#include <vespa/config/common/exceptions.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <thread> - -#include <vespa/log/bufferedlogger.h> -LOG_SETUP(".bucketmover"); - -namespace storage::bucketmover { - -BucketMover::BucketMover(const config::ConfigUri & configUri, - ServiceLayerComponentRegister& reg) - : StorageLink("Bucket mover"), - Runnable(), - framework::HtmlStatusReporter("diskbalancer", "Disk balancer"), - _component(reg, "diskbalancer"), - _config(new vespa::config::content::core::StorBucketmoverConfig()), - _cycleCount(0), - _nextRun(0), - _configFetcher(configUri.getContext()), - _diskDistribution(currentDiskDistribution()), - _maxSleepTime(60 * 60) -{ - if (!configUri.empty()) { - using vespa::config::content::core::StorBucketmoverConfig; - _configFetcher.subscribe<StorBucketmoverConfig>(configUri.getConfigId(), this); - _configFetcher.start(); - } - _component.registerStatusPage(*this); -} - -BucketMover::~BucketMover() -{ - if (_thread) { - LOG(error, "BucketMover deleted without calling close() first"); - onClose(); - } - closeNextLink(); -} - - -void BucketMover::onDoneInit() -{ - framework::MilliSecTime maxProcessingTime(60 * 1000); - framework::MilliSecTime waitTime(_maxSleepTime * 1000); - _thread = _component.startThread(*this, maxProcessingTime, waitTime); -} - -void -BucketMover::onClose() -{ - // Avoid getting config during shutdown - _configFetcher.close(); - // Close thread to ensure we don't send anything more down after - if (_thread) { - _thread->interruptAndJoin(&_wait); - LOG(debug, "Bucket mover worker thread closed."); - _thread.reset(); - } -} - -void -BucketMover::signal() -{ - vespalib::MonitorGuard monitor(_wait); - monitor.signal(); -} - -framework::SecondTime -BucketMover::calculateWaitTimeOfNextRun() const -{ - // _wait lock should have been taken by caller - - // If we haven't tried running at all, run fast to get statistics - if (_history.empty()) { - return framework::SecondTime(_config->minimumRecheckIntervalInSeconds); - } - - // If we have a previous run, assuming our situation haven't changed - // much from that one. Use it to calculate time. - const RunStatistics& lastRun(_history.front()); - - // If there are few buckets in wrong place, don't bother rechecking - // often. - if (lastRun.getWronglyPlacedRatio() < 0.01) { - return framework::SecondTime(_config->maximumRecheckIntervalInSeconds); - } - - // If a disk was disabled, wait for a good while. - for (uint32_t i = 0; i < lastRun._diskData.size(); ++i) { - if (lastRun._diskData[i]._diskDisabled) { - return framework::SecondTime(_config->maximumRecheckIntervalInSeconds / 2); - } - } - - return framework::SecondTime(_config->minimumRecheckIntervalInSeconds); -} - -void -BucketMover::startNewRun() -{ - // If not in a run but time to start another one, do so - LOG(debug, "Starting new move cycle at time %s.", - _component.getClock().getTimeInSeconds().toString().c_str()); - // TODO consider if we should invoke bucket moving across all bucket spaces. Not likely to ever be needed. - // If so, we have to spawn off an individual Run per space, as it encompasses - // both a (disk) distribution and a bucket database. - _currentRun = std::make_unique<bucketmover::Run>( - _component.getBucketSpaceRepo().get(document::FixedBucketSpaces::default_space()), - *_component.getStateUpdater().getReportedNodeState(), - _component.getIndex(), - _component.getClock()); -} - -void -BucketMover::queueNewMoves() -{ - // If we have too few pending, send some new moves, if there are more - // moves to perform. - while (_pendingMoves.size() < uint32_t(_config->maxPending)) { - Move nextMove = _currentRun->getNextMove(); - - // If no more moves to do, stop attempting to send more. - if (!nextMove.isDefined()) { - break; - } - _pendingMoves.push_back(nextMove); - auto cmd = std::make_shared<BucketDiskMoveCommand>( - nextMove.getBucket(), nextMove.getSourceDisk(), nextMove.getTargetDisk()); - cmd->setPriority(nextMove.getPriority()); - _newMoves.push_back(cmd); - } -} - -void -BucketMover::finishCurrentRun() -{ - RunStatistics stats = _currentRun->getStatistics(); - if (_currentRun->aborted()) { - LOG(debug, "Completed aborted bucket move run: %s", - stats.toString().c_str()); - } else { - // If current run is completed, note so in log, and move - // run to history track. - LOG(debug, "Completed bucket move run: %s", - stats.toString().c_str()); - - _history.push_front(stats); - if (_history.size() > uint32_t(_config->maxHistorySize)) { - _history.pop_back(); - } - _nextRun = _component.getClock().getTimeInSeconds() + - calculateWaitTimeOfNextRun(); - } - - _currentRun.reset(); - ++_cycleCount; -} - -void -BucketMover::sendNewMoves() -{ - for (auto& move : _newMoves) { - LOG(debug, "Moving bucket: %s", move->toString().c_str()); - sendDown(move); - - // Be able to sleep a bit between moves for debugging to see - // what is happening. (Cannot use wait() here as reply of - // message sent will signal the monitor) - if (_config->operationDelay != 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(_config->operationDelay)); - } - } - - _newMoves.clear(); -} - -bool -BucketMover::tick() -{ - { - vespalib::MonitorGuard monitor(_wait); - - framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); - - if (!_currentRun) { - if (currentTime >= _nextRun) { - startNewRun(); - } else { - return false; - } - } - - queueNewMoves(); - - if (_newMoves.empty()) { - if (_pendingMoves.empty()) { - finishCurrentRun(); - return true; - } else { - return false; - } - } - } - - // Send delayed after monitor has been unlocked, such that - // incoming responses can grab lock. (Response might come back - // in this thread on errors) - sendNewMoves(); - return true; -} - -void -BucketMover::run(framework::ThreadHandle& thread) -{ - while (!thread.interrupted()) { - thread.registerTick(framework::PROCESS_CYCLE); - if (!tick()) { - vespalib::MonitorGuard monitor(_wait); - monitor.wait(1000); - } - } -} - -void -BucketMover::configure(std::unique_ptr<vespa::config::content::core::StorBucketmoverConfig> config) -{ - vespalib::MonitorGuard monitor(_wait); - if (config->minimumRecheckIntervalInSeconds < 0) { - throw config::InvalidConfigException( - "Minimum recheck interval must be a positive value", - VESPA_STRLOC); - } - if (config->maximumRecheckIntervalInSeconds - < config->minimumRecheckIntervalInSeconds) { - throw config::InvalidConfigException( - "Maximum recheck interval must be equal or greater " - "to minimum recheck interval", - VESPA_STRLOC); - } - if (config->bucketIterationChunk < 1) { - throw config::InvalidConfigException( - "Bucket iteration chunk must be a positive number", - VESPA_STRLOC); - } - if (config->maxTargetFillRateAboveAverage < 0 - || config->maxTargetFillRateAboveAverage > 1.0) - { - throw config::InvalidConfigException( - "Max target fill rate above average must be in the range 0-1", - VESPA_STRLOC); - } - if (config->maxPending < 1) { - throw config::InvalidConfigException( - "Cannot have less than 1 max pending", VESPA_STRLOC); - } - if (config->maxHistorySize < 1) { - throw config::InvalidConfigException( - "Cannot have less than 1 max history size", VESPA_STRLOC); - } - if (config->operationDelay > 0) { - LOG(warning, "Operation delay debug option enabled. Slows down bucket " - "moving. Should only be used in testing where we want to " - "slow down the operation to manually inspect it during " - "the run."); - } - _config = std::move(config); - while (_history.size() > uint32_t(_config->maxHistorySize)) { - _history.pop_back(); - } -} - -bool -BucketMover::onInternalReply( - const std::shared_ptr<api::InternalReply>& internalReply) -{ - // We only care about move disk bucket replies - auto reply = std::dynamic_pointer_cast<BucketDiskMoveReply>(internalReply); - if (!reply) { - return false; - } - - // Warn if we see move replies outside of a run. Should not be possible. - vespalib::MonitorGuard monitor(_wait); - if (!_currentRun) { - LOG(warning, "Got a bucket disk move reply while no run is active. " - "This should not happen, as runs should stay active until " - "all requests are answered."); - return true; - } - // Match move against pending ones - Move move; - for (auto it = _pendingMoves.begin(); it != _pendingMoves.end(); ++it) { - if (it->getBucket() == reply->getBucket() - && it->getSourceDisk() == reply->getSrcDisk() - && it->getTargetDisk() == reply->getDstDisk()) - { - move = *it; - _pendingMoves.erase(it); - break; - } - } - // Warn if it wasn't supposed to be active - if (!move.isDefined()) { - LOG(warning, "Got a bucket disk move reply which wasn't registered " - "as pending. This should not happen."); - return true; - } - // Tag move completed in run. - if (reply->getResult().success()) { - _currentRun->moveOk(move); - } else if (reply->getResult().getResult() - == api::ReturnCode::BUCKET_NOT_FOUND - || reply->getResult().getResult() - == api::ReturnCode::BUCKET_DELETED) - { - _currentRun->moveFailedBucketNotFound(move); - } else { - _currentRun->moveFailed(move); - LOGBP(debug, "Failed %s: %s", - move.toString().c_str(), reply->getResult().toString().c_str()); - } - monitor.broadcast(); - return true; -} - -// TODO if we start supporting disk moves for other spaces than the default space -// we also have to check all disk distributions here. -void -BucketMover::storageDistributionChanged() -{ - // Verify that the actual disk distribution changed, if not ignore - lib::Distribution::DiskDistribution newDistr(currentDiskDistribution()); - - if (_diskDistribution == newDistr) { - return; - } - - vespalib::MonitorGuard monitor(_wait); - if (_currentRun) { - LOG(info, "Aborting bucket mover run as disk distribution changed " - "from %s to %s.", - lib::Distribution::getDiskDistributionName(_diskDistribution).c_str(), - lib::Distribution::getDiskDistributionName(newDistr).c_str()); - _currentRun->abort(); - } else { - LOG(info, "Regathering state as disk distribution changed " - "from %s to %s.", - lib::Distribution::getDiskDistributionName(_diskDistribution).c_str(), - lib::Distribution::getDiskDistributionName(newDistr).c_str()); - } - _diskDistribution = newDistr; - _nextRun = framework::SecondTime(0); -} - -lib::Distribution::DiskDistribution BucketMover::currentDiskDistribution() const { - auto distribution = _component.getBucketSpaceRepo().get(document::FixedBucketSpaces::default_space()).getDistribution(); - return distribution->getDiskDistribution(); -} - -bool BucketMover::isWorkingOnCycle() const { - vespalib::MonitorGuard monitor(_wait); - return (_currentRun.get() != nullptr); -} - -uint32_t BucketMover::getCycleCount() const { - vespalib::MonitorGuard monitor(_wait); - return _cycleCount; -} - -void -BucketMover::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - vespalib::MonitorGuard monitor(_wait); - out << "BucketMover() {"; - if (_currentRun) { - out << "\n" << indent << " "; - _currentRun->print(out, verbose, indent + " "); - } else { - out << "\n" << indent << " No current run."; - } - if (verbose && !_history.empty()) { - out << "\n" << indent << " History:"; - for (auto& entry : _history) { - out << "\n" << indent << " "; - entry.print(out, true, indent + " "); - } - } - out << "\n" << indent << "}"; -} - -void -BucketMover::reportHtmlStatus(std::ostream& out, - const framework::HttpUrlPath&) const -{ - vespalib::MonitorGuard monitor(_wait); - if (_history.empty()) { - out << "<h2>Status after last run</h2>\n"; - out << "<p>No run completed yet. Current status unknown.</p>\n"; - } else { - printCurrentStatus(out, *_history.begin()); - } - out << "<h2>Current move cycle</h2>\n"; - if (_currentRun) { - printRunHtml(out, *_currentRun); - if (_currentRun->getPendingMoves().empty()) { - out << "<blockquote>No pending moves.</blockquote>\n"; - } else { - out << "<blockquote>Pending bucket moves:<ul>\n"; - for (auto& entry : _currentRun->getPendingMoves()) { - out << "<li>" << entry << "</li>\n"; - } - out << "</ul></blockquote>\n"; - } - } else { - out << "<p>\n" - << "No bucket move cycle currently running. "; - framework::SecondTime currentTime( - _component.getClock().getTimeInSeconds()); - if (_nextRun <= currentTime) { - if (_thread) { - out << "Next run to start immediately."; - // Wake up thread, so user sees it starts immediately :) - monitor.signal(); - } else { - out << "Waiting for node to finish initialization before " - << "starting run."; - } - } else { - out << "Next run scheduled to run"; - framework::SecondTime diff(_nextRun - currentTime); - if (diff < framework::SecondTime(24 * 60 * 60)) { - out << " in " << diff.toString(framework::DIFFERENCE); - } else { - out << " at time " << _nextRun; - } - out << "."; - } - out << "\n</p>\n"; - } - if (!_history.empty()) { - out << "<h2>Statistics from previous bucket mover cycles</h2>\n"; - for (auto& entry : _history) { - printRunStatisticsHtml(out, entry); - } - } -} - -void -BucketMover::printCurrentStatus(std::ostream& out, - const RunStatistics& rs) const -{ - framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); - out << "<h2>Status after last run (" - << (currentTime - rs._endTime).toString(framework::DIFFERENCE) - << " ago)</h2>\n" - << "<p>Disk distribution: " - << lib::Distribution::getDiskDistributionName(_diskDistribution) - << "</p>\n"; - out << "<p>This is the status from the last completed bucket database scan " - << "done by the bucket mover. After starting storage, or after " - << "configuration changes, a single scan is always done without " - << "actually attempting to move anything, just to get status updated " - << "quickly. During a move cycle, the data shown for the current cycle " - << "will be more recently updated, but will only represent a part of " - << "the bucket database.</p>\n"; - HtmlTable table("Disk"); - table.addColumnHeader("Real partition byte usage", 3); - ByteSizeColumn diskSpaceUsed("Used", &table); - ByteSizeColumn diskSpaceTotal("Total", &table); - DoubleColumn diskSpaceFillRate("Fill rate", " %", &table); - diskSpaceFillRate.addColorLimit(85, Column::LIGHT_GREEN); - diskSpaceFillRate.addColorLimit(95, Column::LIGHT_YELLOW); - diskSpaceFillRate.addColorLimit(100, Column::LIGHT_RED); - diskSpaceFillRate.setTotalAsAverage(); - table.addColumnHeader("Buckets in directory", 2); - LongColumn bucketCount("Count", "", &table); - PercentageColumn bucketCountPart("Part", 0, &table); - table.addColumnHeader("Total document size directory", 2); - ByteSizeColumn documentSize("Size", &table); - PercentageColumn documentSizePart("Part", 0, &table); - table.addColumnHeader("Buckets on correct disk", 2); - LongColumn bucketsCorrectDisk("Count", "", &table); - DoubleColumn bucketsCorrectDiskPart("Part", " %", &table); - bucketsCorrectDiskPart.setTotalAsAverage(); - bucketsCorrectDiskPart.addColorLimit(95, Column::LIGHT_YELLOW); - bucketsCorrectDiskPart.addColorLimit(100, Column::LIGHT_GREEN); - for (uint32_t i=0; i<rs._diskData.size(); ++i) { - table.addRow(i); - // Ignore disks down - bucketCount[i] = rs.getBucketCount(i, true); - bucketCountPart[i] = bucketCount[i]; - documentSize[i] = rs._diskData[i]._bucketSize; - documentSizePart[i] = documentSize[i]; - bucketsCorrectDisk[i] = rs.getBucketCount(i, false); - bucketsCorrectDiskPart[i] = 100.0 * rs.getBucketCount(i, false) - / rs.getBucketCount(i, true); - } - table.addTotalRow("Total"); - table.print(out); - - MATRIX_PRINT("Buckets on wrong disk", _bucketsLeftOnWrongDisk, rs); -} - -void -BucketMover::printRunHtml(std::ostream& out, const bucketmover::Run& runner) const -{ - printRunStatisticsHtml(out, runner.getStatistics()); -} - -void -BucketMover::printRunStatisticsHtml(std::ostream& out, - const RunStatistics& rs) const -{ - rs.print(out, true, ""); -} - -} diff --git a/storage/src/vespa/storage/bucketmover/bucketmover.h b/storage/src/vespa/storage/bucketmover/bucketmover.h deleted file mode 100644 index 57829eb91bd..00000000000 --- a/storage/src/vespa/storage/bucketmover/bucketmover.h +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::BucketMover - * @ingroup storageserver - * - * @brief This class moves buckets between disks for reducing node skew. Highly - * inspired from BucketIntegrityChecker. - * - * It uses DiskMonitor class to monitor disk info (space available, space used, - * etc), but also to monitor the number of pending moves for each disk. - * It also uses BucketMoverHeuristic class to decide on which buckets should be - * moved and to what disk. - * - * @version $Id: - */ - -#pragma once - -#include "run.h" -#include <vespa/storage/common/storagelink.h> -#include <vespa/storage/config/config-stor-bucketmover.h> -#include <vespa/storage/common/servicelayercomponent.h> -#include <vespa/storageframework/generic/status/htmlstatusreporter.h> -#include <vespa/storageapi/message/bucket.h> -#include <vespa/vdslib/distribution/distribution.h> -#include <vespa/config/helper/ifetchercallback.h> -#include <vespa/config/subscription/configuri.h> -#include <vespa/config/config.h> - -namespace storage { - -class BucketDiskMoveCommand; - -namespace bucketmover { - -class BucketMover : public StorageLink, - private framework::Runnable, - public framework::HtmlStatusReporter, - private config::IFetcherCallback<vespa::config::content::core::StorBucketmoverConfig> -{ - ServiceLayerComponent _component; - std::unique_ptr<vespa::config::content::core::StorBucketmoverConfig> _config; - uint32_t _cycleCount; - framework::SecondTime _nextRun; - std::unique_ptr<bucketmover::Run> _currentRun; - std::list<Move> _pendingMoves; - std::list<std::shared_ptr<BucketDiskMoveCommand> > _newMoves; - std::list<RunStatistics> _history; - vespalib::Monitor _wait; - config::ConfigFetcher _configFetcher; - lib::Distribution::DiskDistribution _diskDistribution; - uint32_t _maxSleepTime; - framework::Thread::UP _thread; - -public: - BucketMover(const config::ConfigUri & configUri, ServiceLayerComponentRegister&); - ~BucketMover(); - - void onDoneInit() override; - void onClose() override; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - bool isWorkingOnCycle() const; - uint32_t getCycleCount() const; - void signal(); - framework::SecondTime getNextRunTime() const { return _nextRun; } - - // Useful for unit testing - vespa::config::content::core::StorBucketmoverConfig& getConfig() { return *_config; } - RunStatistics& getLastRunStats() { return *_history.begin(); } - bool tick(); - void finishCurrentRun(); - -private: - friend struct BucketMoverTest; - - void startNewRun(); - void queueNewMoves(); - void sendNewMoves(); - - void configure(std::unique_ptr<vespa::config::content::core::StorBucketmoverConfig>) override; - void run(framework::ThreadHandle&) override; - bool onInternalReply(const std::shared_ptr<api::InternalReply>&) override; - void storageDistributionChanged() override; - lib::Distribution::DiskDistribution currentDiskDistribution() const; - - framework::SecondTime calculateWaitTimeOfNextRun() const; - - void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; - void printCurrentStatus(std::ostream&, const RunStatistics&) const; - void printRunHtml(std::ostream&, const bucketmover::Run&) const; - void printRunStatisticsHtml(std::ostream&, const RunStatistics&) const; -}; - -} // bucketmover -} // storage diff --git a/storage/src/vespa/storage/bucketmover/move.cpp b/storage/src/vespa/storage/bucketmover/move.cpp deleted file mode 100644 index 12b38980818..00000000000 --- a/storage/src/vespa/storage/bucketmover/move.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "move.h" -#include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <ostream> - -namespace storage { -namespace bucketmover { - -Move::Move() - : _sourceDisk(0), - _targetDisk(0), - _bucket(), - _totalDocSize(0), - _priority(255) -{ -} - -Move::Move(uint16_t source, uint16_t target, const document::Bucket& bucket, - uint32_t totalDocSize) - : _sourceDisk(source), - _targetDisk(target), - _bucket(bucket), - _totalDocSize(totalDocSize), - _priority(255) -{ -} - -void -Move::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - (void) verbose; (void) indent; - if (!isDefined()) { - out << "Move(none)"; - return; - } - out << "Move(" << _bucket << ", " << _sourceDisk << " -> " << _targetDisk - << ", pri " << (uint16_t) _priority << ")"; -} - -} // bucketmover -} // storage diff --git a/storage/src/vespa/storage/bucketmover/move.h b/storage/src/vespa/storage/bucketmover/move.h deleted file mode 100644 index 92d05e4e0ae..00000000000 --- a/storage/src/vespa/storage/bucketmover/move.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::bucketmover::Move - * \ingroup bucketmover - * - * \brief Class representing a bucket move between disks. - */ - -#pragma once - -#include <vespa/document/bucket/bucket.h> -#include <vespa/vespalib/util/printable.h> - -namespace storage { -namespace bucketmover { - -class Move : public vespalib::Printable { - uint16_t _sourceDisk; - uint16_t _targetDisk; - document::Bucket _bucket; - uint32_t _totalDocSize; - uint8_t _priority; - -public: - Move(); - Move(uint16_t source, uint16_t target, const document::Bucket& bucket, - uint32_t totalDocSize); - - /** False if invalid move. (Empty constructor) Indicates end of run. */ - bool isDefined() const { return (_bucket.getBucketId().getRawId() != 0); } - - // Only valid to call if move is defined - uint16_t getSourceDisk() const { return _sourceDisk; } - uint16_t getTargetDisk() const { return _targetDisk; } - const document::Bucket& getBucket() const { return _bucket; } - uint8_t getPriority() const { return _priority; } - uint32_t getTotalDocSize() const { return _totalDocSize; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} // bucketmover -} // storage - diff --git a/storage/src/vespa/storage/bucketmover/run.cpp b/storage/src/vespa/storage/bucketmover/run.cpp deleted file mode 100644 index 22bcfa55f15..00000000000 --- a/storage/src/vespa/storage/bucketmover/run.cpp +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "run.h" -#include <vespa/storage/bucketdb/storbucketdb.h> -#include <vespa/storage/bucketdb/lockablemap.hpp> -#include <vespa/storageframework/generic/clock/clock.h> -#include <iomanip> - -#include <vespa/log/log.h> -LOG_SETUP(".bucketmover.run"); - -namespace storage { -namespace bucketmover { - -Run::Run(ContentBucketSpace& bucketSpace, - const lib::NodeState& nodeState, - uint16_t nodeIndex, - framework::Clock& clock) - : _bucketSpace(bucketSpace), - _distribution(bucketSpace.getDistribution()), - _nodeState(nodeState), - _nodeIndex(nodeIndex), - _entries(), - _iterationDone(false), - _statistics(_distribution->getDiskDistribution(), clock, nodeState), - _aborted(false) -{ -} - -namespace { - struct BucketIterator { - document::BucketSpace _iteratedBucketSpace; - const lib::Distribution& _distribution; - const lib::NodeState& _nodeState; - RunStatistics& _statistics; - std::list<Move>& _entries; - uint16_t _nodeIndex; - uint32_t _maxBucketsToIterateAtOnce; - uint32_t _bucketsVisited; - document::BucketId _firstBucket; - - BucketIterator(document::BucketSpace iteratedBucketSpace, - const lib::Distribution& d, const lib::NodeState& ns, - uint16_t nodeIndex, RunStatistics& stats, - std::list<Move>& entries) - : _iteratedBucketSpace(iteratedBucketSpace), - _distribution(d), - _nodeState(ns), - _statistics(stats), - _entries(entries), - _nodeIndex(nodeIndex), - _maxBucketsToIterateAtOnce(10000), - _bucketsVisited(0), - _firstBucket(stats._lastBucketVisited) - { - } - - StorBucketDatabase::Decision - operator()(document::BucketId::Type revId, - StorBucketDatabase::Entry& entry) - { - document::BucketId bucketId(document::BucketId::keyToBucketId(revId)); - if (bucketId == _firstBucket) { - return StorBucketDatabase::CONTINUE; - } - uint16_t idealDisk = _distribution.getIdealDisk( - _nodeState, _nodeIndex, bucketId, - lib::Distribution::IDEAL_DISK_EVEN_IF_DOWN); - RunStatistics::DiskData& diskData( - _statistics._diskData[entry.disk]); - bool idealDiskDown( - _statistics._diskData[idealDisk]._diskDisabled); - if (entry.disk == idealDisk || idealDiskDown) { - diskData._bucketSize += entry.getBucketInfo().getTotalDocumentSize(); - ++diskData._bucketsFoundOnCorrectDisk; - } else { - document::Bucket bucket(_iteratedBucketSpace, bucketId); - _entries.push_back(Move( - entry.disk, idealDisk, bucket, entry.getBucketInfo().getTotalDocumentSize())); - } - _statistics._lastBucketVisited = bucketId; - if (++_bucketsVisited >= _maxBucketsToIterateAtOnce) { - return StorBucketDatabase::ABORT; - } - return StorBucketDatabase::CONTINUE; - } - }; -} - -Move -Run::getNextMove() -{ - if (_aborted) { - LOG(debug, "Run aborted. Returning undefined move."); - return Move(); - } - if (_iterationDone) { - LOG(debug, "Run completed. End time set. Returning undefined move."); - return Move(); - } - while (true) { - // Process cached entries until we either found one to move, or - // we have no more - while (!_entries.empty()) { - Move e(_entries.front()); - _entries.pop_front(); - - if (!_statistics._diskData[e.getTargetDisk()]._diskDisabled) { - _pending.push_back(e); - _statistics._lastBucketProcessed = e.getBucket(); // Only used for printing - _statistics._lastBucketProcessedTime = _statistics._clock->getTimeInSeconds(); - return e; - } - } - - // Cache more entries - BucketIterator it(_bucketSpace.bucketSpace(), *_distribution, - _nodeState, _nodeIndex, _statistics, _entries); - _bucketSpace.bucketDatabase().all(it, "bucketmover::Run", _statistics._lastBucketVisited.toKey()); - if (it._bucketsVisited == 0) { - _iterationDone = true; - if (_pending.empty()) { - finalize(); - } - LOG(debug, "Last bucket visited. Done iterating buckets in run."); - return Move(); - } - } -} - -void -Run::finalize() -{ - _statistics._endTime = _statistics._clock->getTimeInSeconds(); -} - -void -Run::removePending(Move& move) -{ - bool foundPending = false; - for (auto it = _pending.begin(); it != _pending.end(); ++it) { - if (it->getBucket() == move.getBucket()) { - _pending.erase(it); - foundPending = true; - break; - } - } - if (!foundPending) { - LOG(warning, "Got answer for %s that was not in the pending list.", - move.getBucket().toString().c_str()); - return; - } - if (_iterationDone && _pending.empty()) { - finalize(); - } -} - -void -Run::moveOk(Move& move) -{ - ++_statistics._diskData[move.getSourceDisk()][move.getTargetDisk()] - ._bucketsMoved; - removePending(move); - uint32_t size = move.getTotalDocSize(); - - _statistics._diskData[move.getSourceDisk()]._bucketSize -= size; - _statistics._diskData[move.getTargetDisk()]._bucketSize += size; -} - -void -Run::moveFailedBucketNotFound(Move& move) -{ - ++_statistics._diskData[move.getSourceDisk()][move.getTargetDisk()] - ._bucketsNotFoundAtExecutionTime; - removePending(move); -} - -void -Run::moveFailed(Move& move) -{ - ++_statistics._diskData[move.getSourceDisk()][move.getTargetDisk()] - ._bucketsFailedMoving; - _statistics._diskData[move.getTargetDisk()]._diskDisabled = true; - removePending(move); -} - -void -Run::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "Run("; - if (_aborted) { - out << "Aborted"; - } else if (_statistics._endTime.isSet()) { - if (_entries.empty()) { - out << "Completed"; - } else { - out << "Iteration done"; - } - } - out << ") {\n" << indent << " "; - _statistics.print(out, verbose, indent + " "); - if (!_entries.empty()) { - out << "\n" << indent << " Pending possible moves:"; - uint32_t i = 0; - for (std::list<Move>::const_iterator it = _entries.begin(); - it != _entries.end() && ++i <= 10; ++it) - { - out << "\n" << indent << " " << *it; - } - uint32_t size = _entries.size(); - if (size > 10) { - out << "\n" << indent << " ... and " << (size - 10) - << " more."; - } - } - if (!_statistics._endTime.isSet()) { - out << "\n" << indent << " Bucket iterator: " - << _statistics._lastBucketVisited; - } - out << "\n" << indent << "}"; -} - -} // bucketmover -} // storage diff --git a/storage/src/vespa/storage/bucketmover/run.h b/storage/src/vespa/storage/bucketmover/run.h deleted file mode 100644 index 914781eec30..00000000000 --- a/storage/src/vespa/storage/bucketmover/run.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::bucketmover::Run - * \ingroup storageserver - * - * \brief The run class takes care of creating operations for a single run. - * - * The run class keeps a matrix that keeps track of up to a max number of - * buckets that are located on the wrong disk. The first index of the matrix is - * the source disk and the second index is the ideal disk. - * - * The Run does not care about about pending or wait for that. The caller of - * getNextMove() have to regulate max pending for themselevs. - */ - -#pragma once - - -#include "move.h" -#include "runstatistics.h" -#include <vespa/storage/common/content_bucket_space.h> -#include <vespa/vdslib/distribution/distribution.h> -#include <vespa/vdslib/state/nodestate.h> -#include <list> -#include <map> - -namespace storage { - -class MountPointList; -class PartitionMonitor; -class StorBucketDatabase; -class Clock; - -namespace bucketmover { - -class Run : public document::Printable { - ContentBucketSpace& _bucketSpace; - std::shared_ptr<const lib::Distribution> _distribution; - lib::NodeState _nodeState; - uint16_t _nodeIndex; - std::list<Move> _entries; - std::list<Move> _pending; - bool _iterationDone; - RunStatistics _statistics; - bool _aborted; - std::map<uint16_t, bool> _diskDisabled; - -public: - Run(const Run &) = delete; - Run & operator = (const Run &) = delete; - Run(ContentBucketSpace& bucketSpace, - const lib::NodeState&, - uint16_t nodeIndex, - framework::Clock&); - - /** - * If disk distribution change during runs, they get aborted. We want to - * track this in run, as we want run to exist until all pending requests - * have been answered. - */ - void abort() { _aborted = true; } - bool aborted() { return _aborted; } - - /** - * Get next move does the following: - * - Sort disks in order of fillrate. (PartitionMonitor keeps cache, so - * this will only stat once in a while) - * - If the matrix contains a possible move from above average fill rate - * to below average fill rate, do that move. Prioritizing moving away - * from fullest disk. - * - Otherwise, continue visiting bucket database to fill up matrix. - * - If any moves left, do next, prioritizing moving away from fullest - * disk. - * - * @return A Move object. If isDefined() returns false, run is complete. - * The whole database have been iterated through. - */ - Move getNextMove(); - - void moveOk(Move& move); - void moveFailedBucketNotFound(Move& move); - void moveFailed(Move& move); - - const std::list<Move>& getPendingMoves() const { return _pending; } - - RunStatistics& getStatistics() { return _statistics; } - const RunStatistics& getStatistics() const { return _statistics; } - void print(std::ostream&, bool verbose, const std::string& indent) const override; -private: - void removePending(Move&); - void finalize(); -}; - -} // bucketmover -} // storage diff --git a/storage/src/vespa/storage/bucketmover/runstatistics.cpp b/storage/src/vespa/storage/bucketmover/runstatistics.cpp deleted file mode 100644 index 314f04a0d66..00000000000 --- a/storage/src/vespa/storage/bucketmover/runstatistics.cpp +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "runstatistics.h" -#include "htmltable.h" -#include <vespa/storageframework/generic/clock/clock.h> - - -namespace storage::bucketmover { - -RunStatistics::DiskMatrix::DiskMatrix() - : _bucketsMoved(0), - _bucketsFailedMoving(0), - _bucketsLeftOnWrongDisk(0), - _bucketsNotFoundAtExecutionTime(0) -{ -} - -RunStatistics::DiskData::DiskData(uint16_t diskCount) - : _targetDisks(diskCount), - _bucketsFoundOnCorrectDisk(0), - _bucketSize(0), - _diskDisabled(false) -{ -} - -double -RunStatistics::DiskData::getWronglyPlacedRatio() const -{ - uint64_t wrong = 0; - for (uint32_t i=0; i<_targetDisks.size(); ++i) { - wrong += _targetDisks[i]._bucketsLeftOnWrongDisk - + _targetDisks[i]._bucketsFailedMoving; - } - uint64_t total = wrong + _bucketsFoundOnCorrectDisk; - return static_cast<double>(wrong) / total; -} - -RunStatistics::RunStatistics(DiskDistribution d, framework::Clock& clock, - const lib::NodeState& ns) - : _clock(&clock), - _distribution(d), - _lastBucketProcessed(), - _lastBucketVisited(0), - _diskData(ns.getDiskCount(), DiskData(ns.getDiskCount())), - _startTime(_clock->getTimeInSeconds()), - _endTime(0), - _lastBucketProcessedTime(0) -{ - for (uint32_t i=0; i<ns.getDiskCount(); ++i) { - if (!ns.getDiskState(i).getState().oneOf("uis")) { - _diskData[i]._diskDisabled = true; - } - } -} - -void -RunStatistics::print(std::ostream& out, bool verbose, - const std::string& ind) const -{ - (void) verbose; (void) ind; - bool completed(_endTime.isSet()); - framework::SecondTime currentTime = _clock->getTimeInSeconds(); - if (completed) { - out << "<h3>Run from " << _startTime << " to " << _endTime; - } else { - out << "<h3>Run started " - << currentTime.getDiff(_startTime).toString(framework::DIFFERENCE) - << " ago"; - } - out << " with distribution " - << lib::Distribution::getDiskDistributionName(_distribution) - << "</h3>\n<blockquote>"; - if (!completed) { - std::ostringstream progress; - progress << std::fixed << std::setprecision(4) - << (100.0 * getProgress()); - out << "<p>Progress: " << progress.str() << " % "; - if (_lastBucketProcessedTime.isSet()) { - out << "<font color=\"gray\" size=\"-1\">Last move for " - << _lastBucketProcessed << " " - << currentTime.getDiff(_lastBucketProcessedTime) - .toString(framework::DIFFERENCE) - << " ago</font>"; - } - out << "</p>\n"; - } - - HtmlTable table("Disk"); - table.addColumnHeader(completed ? "Buckets in directory after run" - : "Processed buckets in directory", 2); - LongColumn bucketCount("Count", "", &table); - PercentageColumn bucketCountPart("Part", 0, &table); - - table.addColumnHeader(completed - ? "Total document size in directory after run" - : "Total document size of processed buckets in directory", 2); - ByteSizeColumn documentSize("Size", &table); - PercentageColumn documentSizePart("Part", 0, &table); - - table.addColumnHeader(completed ? "Buckets on correct disk after run" - : "Processed buckets on correct disk", 2); - LongColumn bucketsCorrectDisk("Count", "", &table); - DoubleColumn bucketsCorrectDiskPart("Part", " %", &table); - bucketsCorrectDiskPart.setTotalAsAverage(); - bucketsCorrectDiskPart.addColorLimit(95, Column::LIGHT_YELLOW); - bucketsCorrectDiskPart.addColorLimit(100, Column::LIGHT_GREEN); - - for (uint32_t i=0; i<_diskData.size(); ++i) { - table.addRow(i); - if (_diskData[i]._diskDisabled) { - table.setRowHeaderColor(Column::LIGHT_RED); - } - - bucketCount[i] = getBucketCount(i, true); - bucketCountPart[i] = bucketCount[i]; - - documentSize[i] = _diskData[i]._bucketSize; - documentSizePart[i] = documentSize[i]; - - bucketsCorrectDisk[i] = getBucketCount(i, false); - bucketsCorrectDiskPart[i] - = 100.0 * getBucketCount(i, false) / getBucketCount(i, true); - } - table.addTotalRow("Total"); - table.print(out); - - MATRIX_PRINT("Buckets left on wrong disk", _bucketsLeftOnWrongDisk, *this); - MATRIX_PRINT("Buckets moved", _bucketsMoved, *this); - MATRIX_PRINT("Buckets not found at move time", - _bucketsNotFoundAtExecutionTime, *this); - MATRIX_PRINT("Buckets failed moving for other reasons", - _bucketsFailedMoving, *this); - - out << "</blockquote>\n"; -} - -double -RunStatistics::getWronglyPlacedRatio() const -{ - uint64_t wrong = 0, total = 0; - for (uint32_t i=0; i<_diskData.size(); ++i) { - for (uint32_t j=0; j<_diskData.size(); ++j) { - wrong += _diskData[i][j]._bucketsLeftOnWrongDisk - + _diskData[i][j]._bucketsFailedMoving; - } - total += _diskData[i]._bucketsFoundOnCorrectDisk; - } - total += wrong; - return static_cast<double>(wrong) / total; -} - -// FIXME does not cover multiple spaces (but only used for printing) -double -RunStatistics::getProgress() const -{ - if (_endTime.isSet()) return 1.0; - double result = 0; - double weight = 0.5; - uint64_t key = _lastBucketProcessed.getBucketId().toKey(); - for (uint16_t i=0; i<64; ++i) { - uint64_t flag = uint64_t(1) << (63 - i); - if ((key & flag) == flag) { - result += weight; - } - weight /= 2; - } - return result; -} - -uint64_t -RunStatistics::getBucketCount(uint16_t disk, bool includeWrongLocation) const -{ - uint64_t total = 0; - for (uint32_t i=0; i<_diskData.size(); ++i) { - if (disk == i) total += _diskData[i]._bucketsFoundOnCorrectDisk; - for (uint32_t j=0; j<_diskData.size(); ++j) { - if (disk == i) { - if (includeWrongLocation) { - total += _diskData[i][j]._bucketsLeftOnWrongDisk; - total += _diskData[i][j]._bucketsFailedMoving; - } - } else if (disk == j) { - total += _diskData[i][j]._bucketsMoved; - } - } - } - return total; -} - -} diff --git a/storage/src/vespa/storage/bucketmover/runstatistics.h b/storage/src/vespa/storage/bucketmover/runstatistics.h deleted file mode 100644 index 908f345b307..00000000000 --- a/storage/src/vespa/storage/bucketmover/runstatistics.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::bucketmover::RunStatistics - * \ingroup bucketmover - * - * \brief Statistics gathered from a bucket mover cycle. - */ - -#pragma once - -#define MATRIX_PRINT(desc, var, rs) \ -{ \ - bool anyset = false; \ - for (uint32_t i=0; i<(rs)._diskData.size(); ++i) { \ - for (uint32_t j=0; j<(rs)._diskData.size(); ++j) { \ - anyset |= ((rs)._diskData[i][j].var > 0); \ - } \ - } \ - if (anyset) { \ - out << "<h4>" << desc << "</h4>\n"; \ - HtmlTable matrixTable("Source \\ Target"); \ - using LCUP = std::unique_ptr<LongColumn>; \ - std::vector<LCUP> matrixData((rs)._diskData.size()); \ - for (uint32_t i=0; i<(rs)._diskData.size(); ++i) { \ - std::ostringstream index; \ - index << "Disk " << i; \ - matrixData[i].reset(new LongColumn(index.str(), "", &matrixTable));\ - matrixTable.addRow(index.str()); \ - } \ - for (uint32_t i=0; i<(rs)._diskData.size(); ++i) { \ - for (uint32_t j=0; j<(rs)._diskData.size(); ++j) { \ - (*matrixData[j])[i] = (rs)._diskData[i][j].var; \ - } \ - } \ - matrixTable.print(out); \ - } \ -} - -#include <vespa/vdslib/state/nodestate.h> -#include <vespa/vdslib/distribution/distribution.h> -#include <vespa/document/bucket/bucket.h> -#include <vespa/vespalib/util/printable.h> -#include <vespa/storageframework/generic/clock/time.h> - -#include <vector> - -namespace storage::bucketmover { - -struct RunStatistics : public document::Printable { - using DiskDistribution = lib::Distribution::DiskDistribution; - - /** Data kept as targets for moves for each disk. */ - struct DiskMatrix { - uint32_t _bucketsMoved; - uint32_t _bucketsFailedMoving; - uint32_t _bucketsLeftOnWrongDisk; - uint32_t _bucketsNotFoundAtExecutionTime; - - DiskMatrix(); - }; - - /** Data kept per disk. */ - struct DiskData { - std::vector<DiskMatrix> _targetDisks; - uint32_t _bucketsFoundOnCorrectDisk; - uint64_t _bucketSize; - bool _diskDisabled; - - DiskData(uint16_t diskCount); - - DiskMatrix& operator[](uint16_t index) { return _targetDisks[index]; } - const DiskMatrix& operator[](uint16_t index) const - { return _targetDisks[index]; } - double getWronglyPlacedRatio() const; - }; - - framework::Clock* _clock; - DiskDistribution _distribution; - document::Bucket _lastBucketProcessed; - document::BucketId _lastBucketVisited; // Invalid bucket for starting point - std::vector<DiskData> _diskData; - framework::SecondTime _startTime; - framework::SecondTime _endTime; - framework::SecondTime _lastBucketProcessedTime; - - RunStatistics(DiskDistribution, framework::Clock&, const lib::NodeState&); - - double getWronglyPlacedRatio() const; - double getProgress() const; - uint64_t getBucketCount(uint16_t disk, bool includeWrongLocation) const; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} diff --git a/storage/src/vespa/storage/common/bucketmessages.cpp b/storage/src/vespa/storage/common/bucketmessages.cpp index f953428fc8a..d7f75f12013 100644 --- a/storage/src/vespa/storage/common/bucketmessages.cpp +++ b/storage/src/vespa/storage/common/bucketmessages.cpp @@ -106,60 +106,6 @@ std::unique_ptr<api::StorageReply> ReadBucketInfo::makeReply() { return std::make_unique<ReadBucketInfoReply>(*this); } -BucketDiskMoveCommand::BucketDiskMoveCommand(const document::Bucket &bucket, - uint16_t srcDisk, uint16_t dstDisk) - : api::InternalCommand(ID), - _bucket(bucket), - _srcDisk(srcDisk), - _dstDisk(dstDisk) -{ - setPriority(LOW); -} - -BucketDiskMoveCommand::~BucketDiskMoveCommand() = default; - -void -BucketDiskMoveCommand::setBucketId(const document::BucketId& id) -{ - document::Bucket newBucket(_bucket.getBucketSpace(), id); - _bucket = newBucket; -} - -void -BucketDiskMoveCommand::print(std::ostream& out, bool, const std::string&) const { - out << "BucketDiskMoveCommand(" << _bucket.getBucketId() << ", source " << _srcDisk - << ", target " << _dstDisk << ")"; -} - -BucketDiskMoveReply::BucketDiskMoveReply(const BucketDiskMoveCommand& cmd, - const api::BucketInfo& bucketInfo, - uint32_t sourceFileSize, - uint32_t destinationFileSize) - : api::InternalReply(ID, cmd), - _bucket(cmd.getBucket()), - _bucketInfo(bucketInfo), - _fileSizeOnSrc(sourceFileSize), - _fileSizeOnDst(destinationFileSize), - _srcDisk(cmd.getSrcDisk()), - _dstDisk(cmd.getDstDisk()) -{ } - -BucketDiskMoveReply::~BucketDiskMoveReply() = default; - -void -BucketDiskMoveReply::print(std::ostream& out, bool, const std::string&) const -{ - out << "BucketDiskMoveReply(" << _bucket.getBucketId() << ", source " << _srcDisk - << ", target " << _dstDisk << ", " << _bucketInfo << ", " << getResult() << ")"; -} - -std::unique_ptr<api::StorageReply> -BucketDiskMoveCommand::makeReply() -{ - return std::make_unique<BucketDiskMoveReply>(*this); -} - - InternalBucketJoinCommand::InternalBucketJoinCommand(const document::Bucket &bucket, uint16_t keepOnDisk, uint16_t joinFromDisk) : api::InternalCommand(ID), diff --git a/storage/src/vespa/storage/common/bucketmessages.h b/storage/src/vespa/storage/common/bucketmessages.h index fed944fb217..d6ba7915b3f 100644 --- a/storage/src/vespa/storage/common/bucketmessages.h +++ b/storage/src/vespa/storage/common/bucketmessages.h @@ -115,76 +115,6 @@ public: }; /** - * @class BucketDiskMoveCommand - * @ingroup common - * - * @brief Move a given bucket (from src disk to dst disk). - * - * This message is sent continually by the bucket mover. - * Size of the bucket moved is reported back. - */ -class BucketDiskMoveCommand : public api::InternalCommand { - document::Bucket _bucket; - uint16_t _srcDisk; - uint16_t _dstDisk; - -public: - typedef std::shared_ptr<BucketDiskMoveCommand> SP; - static const uint32_t ID = 2012; - - BucketDiskMoveCommand(const document::Bucket &bucket, uint16_t srcDisk, uint16_t dstDisk); - ~BucketDiskMoveCommand(); - - document::Bucket getBucket() const override { return _bucket; } - bool hasSingleBucketId() const override { return true; } - - uint16_t getSrcDisk() const { return _srcDisk; } - uint16_t getDstDisk() const { return _dstDisk; } - - void setBucketId(const document::BucketId& id); - - std::unique_ptr<api::StorageReply> makeReply() override; - - void print(std::ostream& out, bool, const std::string&) const override; -}; - -/** - * @class BucketDiskMoveReply - * @ingroup common - */ -class BucketDiskMoveReply : public api::InternalReply { - document::Bucket _bucket; - api::BucketInfo _bucketInfo; - uint64_t _fileSizeOnSrc; - uint64_t _fileSizeOnDst; - uint16_t _srcDisk; - uint16_t _dstDisk; - -public: - typedef std::shared_ptr<BucketDiskMoveReply> SP; - static const uint32_t ID = 2013; - - BucketDiskMoveReply(const BucketDiskMoveCommand& cmd, - const api::BucketInfo& bucketInfo = api::BucketInfo(), - uint32_t sourceFileSize = 0, - uint32_t destinationFileSize = 0); - ~BucketDiskMoveReply(); - - document::Bucket getBucket() const override { return _bucket; } - bool hasSingleBucketId() const override { return true; } - - const api::BucketInfo& getBucketInfo() const { return _bucketInfo; } - void setFileSizeOnSrc(uint64_t fileSize) { _fileSizeOnSrc = fileSize; } - void setFileSizeOnDst(uint64_t fileSize) { _fileSizeOnDst = fileSize; } - uint64_t getFileSizeOnSrc() const { return _fileSizeOnSrc; } - uint64_t getFileSizeOnDst() const { return _fileSizeOnDst; } - uint16_t getSrcDisk() const { return _srcDisk; } - uint16_t getDstDisk() const { return _dstDisk; } - - void print(std::ostream& out, bool, const std::string&) const override; -}; - -/** * @class InternalBucketJoinCommand * @ingroup common * diff --git a/storage/src/vespa/storage/common/messagebucket.cpp b/storage/src/vespa/storage/common/messagebucket.cpp index 73c0827c05b..6cc94e2a732 100644 --- a/storage/src/vespa/storage/common/messagebucket.cpp +++ b/storage/src/vespa/storage/common/messagebucket.cpp @@ -64,8 +64,6 @@ getStorageMessageBucket(const api::StorageMessage& msg) return static_cast<const ReadBucketList&>(msg).getBucket(); case ReadBucketInfo::ID: return static_cast<const ReadBucketInfo&>(msg).getBucket(); - case BucketDiskMoveCommand::ID: - return static_cast<const BucketDiskMoveCommand&>(msg).getBucket(); case InternalBucketJoinCommand::ID: return static_cast<const InternalBucketJoinCommand&>(msg).getBucket(); case RecheckBucketInfoCommand::ID: diff --git a/storage/src/vespa/storage/frameworkimpl/thread/CMakeLists.txt b/storage/src/vespa/storage/frameworkimpl/thread/CMakeLists.txt index 603ddf26b90..6cc60e53780 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/CMakeLists.txt +++ b/storage/src/vespa/storage/frameworkimpl/thread/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(storage_thread OBJECT SOURCES appkiller.cpp deadlockdetector.cpp + htmltable.cpp DEPENDS AFTER storage_storageconfig diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp index e1b7d9c52ea..fad99e4e5d3 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "deadlockdetector.h" +#include "htmltable.h" #include <vespa/storage/bucketdb/storbucketdb.h> -#include <vespa/storage/bucketmover/htmltable.h> #include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/storage/src/vespa/storage/bucketmover/htmltable.cpp b/storage/src/vespa/storage/frameworkimpl/thread/htmltable.cpp index 7596e221092..7596e221092 100644 --- a/storage/src/vespa/storage/bucketmover/htmltable.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/htmltable.cpp diff --git a/storage/src/vespa/storage/bucketmover/htmltable.h b/storage/src/vespa/storage/frameworkimpl/thread/htmltable.h index 85e9438edcb..85e9438edcb 100644 --- a/storage/src/vespa/storage/bucketmover/htmltable.h +++ b/storage/src/vespa/storage/frameworkimpl/thread/htmltable.h diff --git a/storage/src/vespa/storage/persistence/CMakeLists.txt b/storage/src/vespa/storage/persistence/CMakeLists.txt index 9edc3bc0080..08e98e9a58e 100644 --- a/storage/src/vespa/storage/persistence/CMakeLists.txt +++ b/storage/src/vespa/storage/persistence/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_library(storage_spersistence OBJECT SOURCES bucketownershipnotifier.cpp bucketprocessor.cpp - diskmoveoperationhandler.cpp fieldvisitor.cpp mergehandler.cpp messages.cpp diff --git a/storage/src/vespa/storage/persistence/diskmoveoperationhandler.cpp b/storage/src/vespa/storage/persistence/diskmoveoperationhandler.cpp deleted file mode 100644 index 18877bdf8f7..00000000000 --- a/storage/src/vespa/storage/persistence/diskmoveoperationhandler.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "diskmoveoperationhandler.h" - -#include <vespa/log/log.h> -LOG_SETUP(".persistence.diskmoveoperationhandler"); - -namespace storage { - -DiskMoveOperationHandler::DiskMoveOperationHandler(PersistenceUtil& env, spi::PersistenceProvider& provider) - : _env(env), - _provider(provider) -{ -} - -MessageTracker::UP -DiskMoveOperationHandler::handleBucketDiskMove(BucketDiskMoveCommand& cmd, MessageTracker::UP tracker) -{ - tracker->setMetric(_env._metrics.movedBuckets); - - document::Bucket bucket(cmd.getBucket()); - uint32_t targetDisk(cmd.getDstDisk()); - uint32_t deviceIndex(_env._partition); - - if (cmd.getSrcDisk() != deviceIndex) { - tracker->fail(api::ReturnCode::INTERNAL_FAILURE, - "Tried to move bucket from source disk where it was not located"); - return tracker; - } - if (targetDisk == deviceIndex) { - tracker->fail(api::ReturnCode::INTERNAL_FAILURE, - "Tried to move bucket from and to the same disk"); - return tracker; - } - if (!_env._fileStorHandler.enabled(targetDisk)) { - tracker->fail(api::ReturnCode::ABORTED, "Target disk is not available"); - return tracker; - } - - LOG(debug, "Moving bucket %s from disk %u to disk %u.", - bucket.toString().c_str(), - deviceIndex, targetDisk); - - spi::Bucket from(bucket, spi::PartitionId(deviceIndex)); - - spi::Result result(_provider.move(from, spi::PartitionId(targetDisk), tracker->context())); - if (result.hasError()) { - tracker->fail(api::ReturnCode::INTERNAL_FAILURE, result.getErrorMessage()); - return tracker; - } - - api::BucketInfo bInfo = _env.getBucketInfo(bucket, targetDisk); - uint32_t sourceFileSize = bInfo.getUsedFileSize(); - - { - // Grab bucket lock in bucket database, and update it - // If entry doesn't exist, that means it has just been deleted by - // delete bucket command. If so, it'll be deleted when delete bucket - // is executed. moving queue should move delete command to correct disk - StorBucketDatabase::WrappedEntry entry( - _env.getBucketDatabase(bucket.getBucketSpace()).get( - bucket.getBucketId(), "FileStorThread::onBucketDiskMove", - StorBucketDatabase::LOCK_IF_NONEXISTING_AND_NOT_CREATING)); - - // Move queued operations in bucket to new thread. Hold bucket lock - // while doing it, so filestor manager can't put in other operations - // first, such that operations change order. - _env._fileStorHandler.remapQueueAfterDiskMove(bucket, deviceIndex, targetDisk); - - if (entry.exist()) { - entry->setBucketInfo(bInfo); - entry->disk = targetDisk; - entry.write(); - } - } - - // Answer message, setting extra info such as filesize - tracker->setReply(std::make_shared<BucketDiskMoveReply>(cmd, bInfo, sourceFileSize, sourceFileSize)); - - return tracker; -} - -} // storage diff --git a/storage/src/vespa/storage/persistence/diskmoveoperationhandler.h b/storage/src/vespa/storage/persistence/diskmoveoperationhandler.h deleted file mode 100644 index 9e8d33fc802..00000000000 --- a/storage/src/vespa/storage/persistence/diskmoveoperationhandler.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/storage/common/bucketmessages.h> -#include <vespa/storage/persistence/persistenceutil.h> - -namespace storage { - -class DiskMoveOperationHandler : public Types { - -public: - DiskMoveOperationHandler(PersistenceUtil&, - spi::PersistenceProvider& provider); - - MessageTracker::UP handleBucketDiskMove(BucketDiskMoveCommand&, MessageTracker::UP tracker); - -private: - PersistenceUtil& _env; - spi::PersistenceProvider& _provider; -}; - -} // storage - diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index e23d65f72ac..5627ade2a11 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -658,22 +658,6 @@ FileStorHandlerImpl::remapMessage(api::StorageMessage& msg, const document::Buck } } break; - case BucketDiskMoveCommand::ID: - // Fail bucket not found if op != MOVE - // Fail and log error if op == MOVE - { - api::BucketCommand& cmd(static_cast<api::BucketCommand&>(msg)); - if (cmd.getBucket() == source) { - if (op == MOVE) { - returnCode = api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE, - "Multiple bucket disk move commands pending at the same time " - " towards bucket " + source.toString()); - } else { - returnCode = api::ReturnCode(api::ReturnCode::BUCKET_DELETED, splitOrJoin(op)); - } - } - break; - } case ReadBucketInfo::ID: case RecheckBucketInfoCommand::ID: { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 59ae6b83f56..db3d865faca 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -692,15 +692,6 @@ FileStorManager::onInternal(const shared_ptr<api::InternalCommand>& msg) } return true; } - case BucketDiskMoveCommand::ID: - { - shared_ptr<BucketDiskMoveCommand> cmd(std::static_pointer_cast<BucketDiskMoveCommand>(msg)); - StorBucketDatabase::WrappedEntry entry(mapOperationToDisk(*cmd, cmd->getBucket())); - if (entry.exist()) { - handlePersistenceMessage(cmd, entry->disk); - } - return true; - } case RecheckBucketInfoCommand::ID: { shared_ptr<RecheckBucketInfoCommand> cmd(std::static_pointer_cast<RecheckBucketInfoCommand>(msg)); diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 1368b14077a..8eac55445f9 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -28,7 +28,6 @@ PersistenceThread::PersistenceThread(ServiceLayerComponentRegister& compReg, _spi(provider), _processAllHandler(_env, provider), _mergeHandler(_spi, _env), - _diskMoveHandler(_env, _spi), _bucketOwnershipNotifier(), _flushMonitor(), _closed(false) @@ -724,8 +723,6 @@ PersistenceThread::handleCommandSplitByType(api::StorageCommand& msg, MessageTra return handleReadBucketList(static_cast<ReadBucketList&>(msg), std::move(tracker)); case ReadBucketInfo::ID: return handleReadBucketInfo(static_cast<ReadBucketInfo&>(msg), std::move(tracker)); - case BucketDiskMoveCommand::ID: - return _diskMoveHandler.handleBucketDiskMove(static_cast<BucketDiskMoveCommand&>(msg), std::move(tracker)); case InternalBucketJoinCommand::ID: return handleInternalBucketJoin(static_cast<InternalBucketJoinCommand&>(msg), std::move(tracker)); case RecheckBucketInfoCommand::ID: diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index 773732b5ef1..fcecc963658 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -5,9 +5,9 @@ #include "diskthread.h" #include "processallhandler.h" #include "mergehandler.h" -#include "diskmoveoperationhandler.h" #include "persistenceutil.h" #include "provider_error_wrapper.h" +#include <vespa/storage/common/bucketmessages.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/common/statusmessages.h> @@ -52,7 +52,6 @@ private: spi::PersistenceProvider& _spi; ProcessAllHandler _processAllHandler; MergeHandler _mergeHandler; - DiskMoveOperationHandler _diskMoveHandler; ServiceLayerComponent::UP _component; framework::Thread::UP _thread; std::unique_ptr<BucketOwnershipNotifier> _bucketOwnershipNotifier; diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.cpp b/storage/src/vespa/storage/storageserver/servicelayernode.cpp index 524e7c19d53..1b90e235d20 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernode.cpp @@ -13,7 +13,6 @@ #include <vespa/storage/visiting/visitormanager.h> #include <vespa/storage/bucketdb/bucketmanager.h> #include <vespa/storage/bucketdb/storagebucketdbinitializer.h> -#include <vespa/storage/bucketmover/bucketmover.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> #include <vespa/persistence/spi/exceptions.h> @@ -241,7 +240,6 @@ ServiceLayerNode::createChain() auto* merge_throttler = new MergeThrottler(_configUri, compReg); chain->push_back(StorageLink::UP(merge_throttler)); chain->push_back(StorageLink::UP(new ChangedBucketOwnershipHandler(_configUri, compReg))); - chain->push_back(StorageLink::UP(new bucketmover::BucketMover(_configUri, compReg))); chain->push_back(StorageLink::UP(new StorageBucketDBInitializer( _configUri, _partitions, getDoneInitializeHandler(), compReg))); chain->push_back(StorageLink::UP(new BucketManager(_configUri, _context.getComponentRegister()))); |