diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2020-05-04 10:16:07 +0000 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2020-05-04 14:16:02 +0000 |
commit | 606e302390d51d389ebee8d09e966111f7b40e50 (patch) | |
tree | 68461c948ed75eb840e6b4be9ed67642840a9342 /storage | |
parent | a5a736466f91700bfe7bb72538463aa274ce26e3 (diff) |
Remove deprecated bucket cross-disk move functionality
The notion of multiple disks hasn't been supported since we
removed VDS, and likely won't be in the future either.
Diffstat (limited to 'storage')
35 files changed, 15 insertions, 1919 deletions
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 52423f031e5..e9f2d736b91 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -24,7 +24,6 @@ vespa_define_module( LIBS src/vespa/storage src/vespa/storage/bucketdb - src/vespa/storage/bucketmover src/vespa/storage/common src/vespa/storage/common/hostreporter src/vespa/storage/config @@ -52,7 +51,6 @@ vespa_define_module( TESTS src/tests src/tests/bucketdb - src/tests/bucketmover src/tests/common src/tests/common/hostreporter src/tests/distributor diff --git a/storage/src/tests/bucketmover/.gitignore b/storage/src/tests/bucketmover/.gitignore deleted file mode 100644 index 2f86228e663..00000000000 --- a/storage/src/tests/bucketmover/.gitignore +++ /dev/null @@ -1 +0,0 @@ -storage_bucketmover_gtest_runner_app diff --git a/storage/src/tests/bucketmover/CMakeLists.txt b/storage/src/tests/bucketmover/CMakeLists.txt deleted file mode 100644 index c7a6e8fc21e..00000000000 --- a/storage/src/tests/bucketmover/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(storage_bucketmover_gtest_runner_app TEST - SOURCES - bucketmovertest.cpp - htmltabletest.cpp - gtest_runner.cpp - DEPENDS - storage - storage_testcommon - gtest -) - -vespa_add_test( - NAME storage_bucketmover_gtest_runner_app - COMMAND storage_bucketmover_gtest_runner_app -) diff --git a/storage/src/tests/bucketmover/bucketmovertest.cpp b/storage/src/tests/bucketmover/bucketmovertest.cpp deleted file mode 100644 index e360819c5d7..00000000000 --- a/storage/src/tests/bucketmover/bucketmovertest.cpp +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/storage/bucketdb/storbucketdb.h> -#include <vespa/storage/common/bucketmessages.h> -#include <vespa/storage/bucketmover/bucketmover.h> -#include <vespa/document/test/make_bucket_space.h> -#include <vespa/document/test/make_document_bucket.h> -#include <tests/common/dummystoragelink.h> -#include <tests/common/teststorageapp.h> -#include <vespa/config/common/exceptions.h> -#include <gtest/gtest.h> - -bool debug = false; - -using document::test::makeBucketSpace; -using document::test::makeDocumentBucket; - -namespace storage::bucketmover { - -struct BucketMoverTest : public ::testing::Test { -public: - void SetUp() override; - void TearDown() override; - - std::unique_ptr<TestServiceLayerApp> _node; - std::unique_ptr<ServiceLayerComponent> _component; - std::unique_ptr<BucketMover> _bucketMover; - DummyStorageLink* after; - - void addBucket(const document::BucketId& id, uint16_t idealDiff); -}; - -void -BucketMoverTest::TearDown() -{ - _node.reset(0); -} - -void -BucketMoverTest::SetUp() -{ - try { - _node.reset(new TestServiceLayerApp(DiskCount(4))); - _node->setupDummyPersistence(); - } catch (config::InvalidConfigException& e) { - fprintf(stderr, "%s\n", e.what()); - } - - _component.reset(new ServiceLayerComponent(_node->getComponentRegister(), "foo")); - _bucketMover.reset(new BucketMover("raw:", _node->getComponentRegister())); - after = new DummyStorageLink(); - _bucketMover->push_back(StorageLink::UP(after)); -} - -void -BucketMoverTest::addBucket(const document::BucketId& id, - uint16_t idealDiff) -{ - StorBucketDatabase::WrappedEntry entry( - _component->getBucketDatabase(makeBucketSpace()).get( - id, - "", - StorBucketDatabase::CREATE_IF_NONEXISTING)); - - entry->setBucketInfo(api::BucketInfo(1,1,1)); - - uint16_t idealDisk = _component->getIdealPartition(makeDocumentBucket(id)); - entry->disk = (idealDisk + idealDiff) % _component->getDiskCount(); - entry.write(); -} - -TEST_F(BucketMoverTest, testNormalUsage) -{ - for (uint32_t i = 1; i < 4; ++i) { - addBucket(document::BucketId(16, i), 1); - } - for (uint32_t i = 4; i < 6; ++i) { - addBucket(document::BucketId(16, i), 0); - } - - _bucketMover->open(); - _bucketMover->tick(); - - std::vector<api::StorageMessage::SP> msgs = after->getCommandsOnce(); - EXPECT_EQ( - std::string("BucketDiskMoveCommand(" - "BucketId(0x4000000000000002), source 3, target 2)"), - msgs[0]->toString()); - EXPECT_EQ( - std::string("BucketDiskMoveCommand(" - "BucketId(0x4000000000000001), source 2, target 1)"), - msgs[1]->toString()); - EXPECT_EQ( - std::string("BucketDiskMoveCommand(" - "BucketId(0x4000000000000003), source 1, target 0)"), - msgs[2]->toString()); - - for (uint32_t i = 0; i < 2; ++i) { - after->sendUp(std::shared_ptr<api::StorageMessage>( - ((api::StorageCommand*)msgs[i].get())-> - makeReply().release())); - } - - _bucketMover->tick(); - EXPECT_EQ(0, (int)after->getNumCommands()); - - _bucketMover->finishCurrentRun(); -} - -TEST_F(BucketMoverTest, testMaxPending) -{ - for (uint32_t i = 1; i < 100; ++i) { - addBucket(document::BucketId(16, i), 1); - } - for (uint32_t i = 101; i < 200; ++i) { - addBucket(document::BucketId(16, i), 0); - } - - _bucketMover->open(); - _bucketMover->tick(); - - std::vector<api::StorageMessage::SP> msgs = after->getCommandsOnce(); - // 5 is the max pending default config. - ASSERT_EQ(5, (int)msgs.size()); - - after->sendUp(std::shared_ptr<api::StorageMessage>( - ((api::StorageCommand*)msgs[3].get())-> - makeReply().release())); - - _bucketMover->tick(); - - std::vector<api::StorageMessage::SP> msgs2 = after->getCommandsOnce(); - ASSERT_EQ(1, (int)msgs2.size()); -} - -TEST_F(BucketMoverTest, testErrorHandling) -{ - for (uint32_t i = 1; i < 100; ++i) { - addBucket(document::BucketId(16, i), 1); - } - for (uint32_t i = 101; i < 200; ++i) { - addBucket(document::BucketId(16, i), 0); - } - - _bucketMover->open(); - _bucketMover->tick(); - - std::vector<api::StorageMessage::SP> msgs = after->getCommandsOnce(); - // 5 is the max pending default config. - ASSERT_EQ(5, (int)msgs.size()); - - BucketDiskMoveCommand& cmd = static_cast<BucketDiskMoveCommand&>(*msgs[0]); - uint32_t targetDisk = cmd.getDstDisk(); - - std::unique_ptr<api::StorageReply> reply(cmd.makeReply().release()); - reply->setResult(api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE, "foobar")); - after->sendUp(std::shared_ptr<api::StorageMessage>(reply.release())); - - for (uint32_t i = 1; i < msgs.size(); ++i) { - after->sendUp(std::shared_ptr<api::StorageMessage>( - ((api::StorageCommand*)msgs[i].get())-> - makeReply().release())); - } - - _bucketMover->tick(); - - std::vector<api::StorageMessage::SP> msgs2 = after->getCommandsOnce(); - ASSERT_EQ(5, (int)msgs2.size()); - - for (uint32_t i = 0; i < msgs2.size(); ++i) { - BucketDiskMoveCommand& bdm = static_cast<BucketDiskMoveCommand&>(*msgs2[i]); - EXPECT_NE(bdm.getDstDisk(), targetDisk); - } -} - -} diff --git a/storage/src/tests/bucketmover/gtest_runner.cpp b/storage/src/tests/bucketmover/gtest_runner.cpp deleted file mode 100644 index 3d49ada55a7..00000000000 --- a/storage/src/tests/bucketmover/gtest_runner.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/gtest/gtest.h> - -#include <vespa/log/log.h> -LOG_SETUP("storage_bucketmover_gtest_runner"); - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/storage/src/tests/frameworkimpl/status/CMakeLists.txt b/storage/src/tests/frameworkimpl/status/CMakeLists.txt index 655dff1bd0c..1b49b1bac45 100644 --- a/storage/src/tests/frameworkimpl/status/CMakeLists.txt +++ b/storage/src/tests/frameworkimpl/status/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(storage_status_gtest_runner_app TEST SOURCES gtest_runner.cpp + htmltabletest.cpp statustest.cpp DEPENDS storage diff --git a/storage/src/tests/bucketmover/htmltabletest.cpp b/storage/src/tests/frameworkimpl/status/htmltabletest.cpp index 8791ae1ec8d..132dee55d93 100644 --- a/storage/src/tests/bucketmover/htmltabletest.cpp +++ b/storage/src/tests/frameworkimpl/status/htmltabletest.cpp @@ -1,13 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/bucketmover/htmltable.h> +#include <vespa/storage/frameworkimpl/thread/htmltable.h> #include <gtest/gtest.h> namespace storage { TEST(HtmlTableTest, testPercentageColumn) { - // With total hardcoded to 100 + // With total hardcoded to 100 { HtmlTable table("disk"); PercentageColumn perc("fillrate", 100); @@ -24,15 +24,15 @@ TEST(HtmlTableTest, testPercentageColumn) std::ostringstream ost; table.print(ost); std::string expected( -"<table border=\"1\" cellpadding=\"2\" cellspacing=\"0\">\n" -"<tr><th>disk</th><th>fillrate</th></tr>\n" -"<tr><td>0</td><td bgcolor=\"#a0ffa0\" align=\"right\">30.00 %</td></tr>\n" -"<tr><td>1</td><td bgcolor=\"#ffffa0\" align=\"right\">80.00 %</td></tr>\n" -"<tr><td>2</td><td bgcolor=\"#ffa0a0\" align=\"right\">100.00 %</td></tr>\n" -"</table>\n"); + "<table border=\"1\" cellpadding=\"2\" cellspacing=\"0\">\n" + "<tr><th>disk</th><th>fillrate</th></tr>\n" + "<tr><td>0</td><td bgcolor=\"#a0ffa0\" align=\"right\">30.00 %</td></tr>\n" + "<tr><td>1</td><td bgcolor=\"#ffffa0\" align=\"right\">80.00 %</td></tr>\n" + "<tr><td>2</td><td bgcolor=\"#ffa0a0\" align=\"right\">100.00 %</td></tr>\n" + "</table>\n"); EXPECT_EQ(expected, ost.str()); } - // With automatically gathered total + // With automatically gathered total { HtmlTable table("disk"); PercentageColumn perc("fillrate"); @@ -65,7 +65,7 @@ TEST(HtmlTableTest, testByteSizeColumn) table.addRow(0); table.addRow(1); table.addRow(2); - // Biggest value enforce the denomination + // Biggest value enforce the denomination size[0] = 42123; size[1] = 124123151; size[2] = 6131231; @@ -83,4 +83,4 @@ TEST(HtmlTableTest, testByteSizeColumn) } -} // storage +} // storage
\ No newline at end of file diff --git a/storage/src/tests/persistence/CMakeLists.txt b/storage/src/tests/persistence/CMakeLists.txt index 1e25f38fe2a..e8095109806 100644 --- a/storage/src/tests/persistence/CMakeLists.txt +++ b/storage/src/tests/persistence/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_executable(storage_persistence_gtest_runner_app TEST SOURCES bucketownershipnotifiertest.cpp - diskmoveoperationhandlertest.cpp mergehandlertest.cpp persistencequeuetest.cpp persistencetestutils.cpp diff --git a/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp b/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp deleted file mode 100644 index 371aa88e692..00000000000 --- a/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/storage/persistence/diskmoveoperationhandler.h> -#include <vespa/storage/persistence/messages.h> -#include <tests/persistence/persistencetestutils.h> -#include <vespa/document/test/make_document_bucket.h> - -using document::test::makeDocumentBucket; -using namespace ::testing; - -namespace storage { - -struct DiskMoveOperationHandlerTest : PersistenceTestUtils {}; - -TEST_F(DiskMoveOperationHandlerTest, simple) { - setupDisks(10); - - // Create bucket 16, 4 on disk 3. - { - StorBucketDatabase::WrappedEntry entry( - createBucket(document::BucketId(16, 4))); - entry->disk = 3; - entry.write(); - } - - for (uint32_t i = 0; i < 10; i++) { - doPutOnDisk(3, 4, spi::Timestamp(1000 + i)); - } - - DiskMoveOperationHandler diskMoveHandler(getEnv(3),getPersistenceProvider()); - document::Bucket bucket = makeDocumentBucket(document::BucketId(16, 4)); - auto move = std::make_shared<BucketDiskMoveCommand>(bucket, 3, 4); - spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); - diskMoveHandler.handleBucketDiskMove(*move, createTracker(move, bucket)); - - EXPECT_EQ("BucketId(0x4000000000000004): 10,4", - getBucketStatus(document::BucketId(16,4))); -} - -} diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 811b4d60e15..bf0083311fb 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -334,76 +334,6 @@ TEST_F(FileStorManagerTest, put) { } } -TEST_F(FileStorManagerTest, disk_move) { - setupDisks(2); - - // Setting up manager - DummyStorageLink top; - FileStorManager *manager; - top.push_back(unique_ptr<StorageLink>(manager = - new FileStorManager(config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); - top.open(); - api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); - // Creating a document to test with - Document::SP doc(createDocument( - "some content", "id:crawler:testdoctype1:n=4000:foo").release()); - - document::BucketId bid(16, 4000); - - createBucket(bid, 0); - - // Putting it - { - auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(address); - top.sendDown(cmd); - top.waitForMessages(1, _waitTime); - ASSERT_EQ(1, top.getNumReplies()); - auto reply = std::dynamic_pointer_cast<api::PutReply>(top.getReply(0)); - top.reset(); - ASSERT_TRUE(reply.get()); - EXPECT_EQ(ReturnCode(ReturnCode::OK), reply->getResult()); - EXPECT_EQ(1, reply->getBucketInfo().getDocumentCount()); - } - - { - StorBucketDatabase::WrappedEntry entry( - _node->getStorageBucketDatabase().get(bid, "foo")); - - EXPECT_EQ(0, entry->disk); - EXPECT_EQ( - vespalib::string( - "BucketInfo(crc 0x3538028e, docCount 1, totDocSize 124, " - "ready true, active false)"), - entry->getBucketInfo().toString()); - } - - { - auto cmd = std::make_shared<BucketDiskMoveCommand>(makeDocumentBucket(bid), 0, 1); - - top.sendDown(cmd); - top.waitForMessages(1, _waitTime); - ASSERT_EQ(1, top.getNumReplies()); - auto reply = std::dynamic_pointer_cast<BucketDiskMoveReply>(top.getReply(0)); - top.reset(); - ASSERT_TRUE(reply.get()); - EXPECT_EQ(ReturnCode(ReturnCode::OK), reply->getResult()); - EXPECT_EQ(1, reply->getBucketInfo().getDocumentCount()); - } - - { - StorBucketDatabase::WrappedEntry entry( - _node->getStorageBucketDatabase().get(bid, "foo")); - - EXPECT_EQ(1, entry->disk); - EXPECT_EQ( - vespalib::string( - "BucketInfo(crc 0x3538028e, docCount 1, totDocSize 124, " - "ready true, active false)"), - entry->getBucketInfo().toString()); - } -} - TEST_F(FileStorManagerTest, state_change) { // Setting up manager DummyStorageLink top; diff --git a/storage/src/vespa/storage/CMakeLists.txt b/storage/src/vespa/storage/CMakeLists.txt index d854c603097..100118b5133 100644 --- a/storage/src/vespa/storage/CMakeLists.txt +++ b/storage/src/vespa/storage/CMakeLists.txt @@ -9,7 +9,6 @@ vespa_add_library(storage $<TARGET_OBJECTS:storage_spersistence> $<TARGET_OBJECTS:storage_storageutil> $<TARGET_OBJECTS:storage_visitor> - $<TARGET_OBJECTS:storage_bucketmover> $<TARGET_OBJECTS:storage_thread> $<TARGET_OBJECTS:storage_status> $<TARGET_OBJECTS:storage_component> diff --git a/storage/src/vespa/storage/bucketmover/CMakeLists.txt b/storage/src/vespa/storage/bucketmover/CMakeLists.txt deleted file mode 100644 index 669c13d3bc1..00000000000 --- a/storage/src/vespa/storage/bucketmover/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_bucketmover OBJECT - SOURCES - bucketmover.cpp - htmltable.cpp - move.cpp - run.cpp - runstatistics.cpp - DEPENDS - AFTER - storage_storageconfig -) diff --git a/storage/src/vespa/storage/bucketmover/bucketmover.cpp b/storage/src/vespa/storage/bucketmover/bucketmover.cpp deleted file mode 100644 index 904e8a66c27..00000000000 --- a/storage/src/vespa/storage/bucketmover/bucketmover.cpp +++ /dev/null @@ -1,528 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bucketmover.h" -#include "htmltable.h" -#include <vespa/storage/config/config-stor-server.h> -#include <vespa/storage/common/bucketmessages.h> -#include <vespa/storage/common/content_bucket_space_repo.h> -#include <vespa/storage/common/nodestateupdater.h> -#include <vespa/config/common/exceptions.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <thread> - -#include <vespa/log/bufferedlogger.h> -LOG_SETUP(".bucketmover"); - -namespace storage::bucketmover { - -BucketMover::BucketMover(const config::ConfigUri & configUri, - ServiceLayerComponentRegister& reg) - : StorageLink("Bucket mover"), - Runnable(), - framework::HtmlStatusReporter("diskbalancer", "Disk balancer"), - _component(reg, "diskbalancer"), - _config(new vespa::config::content::core::StorBucketmoverConfig()), - _cycleCount(0), - _nextRun(0), - _configFetcher(configUri.getContext()), - _diskDistribution(currentDiskDistribution()), - _maxSleepTime(60 * 60) -{ - if (!configUri.empty()) { - using vespa::config::content::core::StorBucketmoverConfig; - _configFetcher.subscribe<StorBucketmoverConfig>(configUri.getConfigId(), this); - _configFetcher.start(); - } - _component.registerStatusPage(*this); -} - -BucketMover::~BucketMover() -{ - if (_thread) { - LOG(error, "BucketMover deleted without calling close() first"); - onClose(); - } - closeNextLink(); -} - - -void BucketMover::onDoneInit() -{ - framework::MilliSecTime maxProcessingTime(60 * 1000); - framework::MilliSecTime waitTime(_maxSleepTime * 1000); - _thread = _component.startThread(*this, maxProcessingTime, waitTime); -} - -void -BucketMover::onClose() -{ - // Avoid getting config during shutdown - _configFetcher.close(); - // Close thread to ensure we don't send anything more down after - if (_thread) { - _thread->interruptAndJoin(&_wait); - LOG(debug, "Bucket mover worker thread closed."); - _thread.reset(); - } -} - -void -BucketMover::signal() -{ - vespalib::MonitorGuard monitor(_wait); - monitor.signal(); -} - -framework::SecondTime -BucketMover::calculateWaitTimeOfNextRun() const -{ - // _wait lock should have been taken by caller - - // If we haven't tried running at all, run fast to get statistics - if (_history.empty()) { - return framework::SecondTime(_config->minimumRecheckIntervalInSeconds); - } - - // If we have a previous run, assuming our situation haven't changed - // much from that one. Use it to calculate time. - const RunStatistics& lastRun(_history.front()); - - // If there are few buckets in wrong place, don't bother rechecking - // often. - if (lastRun.getWronglyPlacedRatio() < 0.01) { - return framework::SecondTime(_config->maximumRecheckIntervalInSeconds); - } - - // If a disk was disabled, wait for a good while. - for (uint32_t i = 0; i < lastRun._diskData.size(); ++i) { - if (lastRun._diskData[i]._diskDisabled) { - return framework::SecondTime(_config->maximumRecheckIntervalInSeconds / 2); - } - } - - return framework::SecondTime(_config->minimumRecheckIntervalInSeconds); -} - -void -BucketMover::startNewRun() -{ - // If not in a run but time to start another one, do so - LOG(debug, "Starting new move cycle at time %s.", - _component.getClock().getTimeInSeconds().toString().c_str()); - // TODO consider if we should invoke bucket moving across all bucket spaces. Not likely to ever be needed. - // If so, we have to spawn off an individual Run per space, as it encompasses - // both a (disk) distribution and a bucket database. - _currentRun = std::make_unique<bucketmover::Run>( - _component.getBucketSpaceRepo().get(document::FixedBucketSpaces::default_space()), - *_component.getStateUpdater().getReportedNodeState(), - _component.getIndex(), - _component.getClock()); -} - -void -BucketMover::queueNewMoves() -{ - // If we have too few pending, send some new moves, if there are more - // moves to perform. - while (_pendingMoves.size() < uint32_t(_config->maxPending)) { - Move nextMove = _currentRun->getNextMove(); - - // If no more moves to do, stop attempting to send more. - if (!nextMove.isDefined()) { - break; - } - _pendingMoves.push_back(nextMove); - auto cmd = std::make_shared<BucketDiskMoveCommand>( - nextMove.getBucket(), nextMove.getSourceDisk(), nextMove.getTargetDisk()); - cmd->setPriority(nextMove.getPriority()); - _newMoves.push_back(cmd); - } -} - -void -BucketMover::finishCurrentRun() -{ - RunStatistics stats = _currentRun->getStatistics(); - if (_currentRun->aborted()) { - LOG(debug, "Completed aborted bucket move run: %s", - stats.toString().c_str()); - } else { - // If current run is completed, note so in log, and move - // run to history track. - LOG(debug, "Completed bucket move run: %s", - stats.toString().c_str()); - - _history.push_front(stats); - if (_history.size() > uint32_t(_config->maxHistorySize)) { - _history.pop_back(); - } - _nextRun = _component.getClock().getTimeInSeconds() + - calculateWaitTimeOfNextRun(); - } - - _currentRun.reset(); - ++_cycleCount; -} - -void -BucketMover::sendNewMoves() -{ - for (auto& move : _newMoves) { - LOG(debug, "Moving bucket: %s", move->toString().c_str()); - sendDown(move); - - // Be able to sleep a bit between moves for debugging to see - // what is happening. (Cannot use wait() here as reply of - // message sent will signal the monitor) - if (_config->operationDelay != 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(_config->operationDelay)); - } - } - - _newMoves.clear(); -} - -bool -BucketMover::tick() -{ - { - vespalib::MonitorGuard monitor(_wait); - - framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); - - if (!_currentRun) { - if (currentTime >= _nextRun) { - startNewRun(); - } else { - return false; - } - } - - queueNewMoves(); - - if (_newMoves.empty()) { - if (_pendingMoves.empty()) { - finishCurrentRun(); - return true; - } else { - return false; - } - } - } - - // Send delayed after monitor has been unlocked, such that - // incoming responses can grab lock. (Response might come back - // in this thread on errors) - sendNewMoves(); - return true; -} - -void -BucketMover::run(framework::ThreadHandle& thread) -{ - while (!thread.interrupted()) { - thread.registerTick(framework::PROCESS_CYCLE); - if (!tick()) { - vespalib::MonitorGuard monitor(_wait); - monitor.wait(1000); - } - } -} - -void -BucketMover::configure(std::unique_ptr<vespa::config::content::core::StorBucketmoverConfig> config) -{ - vespalib::MonitorGuard monitor(_wait); - if (config->minimumRecheckIntervalInSeconds < 0) { - throw config::InvalidConfigException( - "Minimum recheck interval must be a positive value", - VESPA_STRLOC); - } - if (config->maximumRecheckIntervalInSeconds - < config->minimumRecheckIntervalInSeconds) { - throw config::InvalidConfigException( - "Maximum recheck interval must be equal or greater " - "to minimum recheck interval", - VESPA_STRLOC); - } - if (config->bucketIterationChunk < 1) { - throw config::InvalidConfigException( - "Bucket iteration chunk must be a positive number", - VESPA_STRLOC); - } - if (config->maxTargetFillRateAboveAverage < 0 - || config->maxTargetFillRateAboveAverage > 1.0) - { - throw config::InvalidConfigException( - "Max target fill rate above average must be in the range 0-1", - VESPA_STRLOC); - } - if (config->maxPending < 1) { - throw config::InvalidConfigException( - "Cannot have less than 1 max pending", VESPA_STRLOC); - } - if (config->maxHistorySize < 1) { - throw config::InvalidConfigException( - "Cannot have less than 1 max history size", VESPA_STRLOC); - } - if (config->operationDelay > 0) { - LOG(warning, "Operation delay debug option enabled. Slows down bucket " - "moving. Should only be used in testing where we want to " - "slow down the operation to manually inspect it during " - "the run."); - } - _config = std::move(config); - while (_history.size() > uint32_t(_config->maxHistorySize)) { - _history.pop_back(); - } -} - -bool -BucketMover::onInternalReply( - const std::shared_ptr<api::InternalReply>& internalReply) -{ - // We only care about move disk bucket replies - auto reply = std::dynamic_pointer_cast<BucketDiskMoveReply>(internalReply); - if (!reply) { - return false; - } - - // Warn if we see move replies outside of a run. Should not be possible. - vespalib::MonitorGuard monitor(_wait); - if (!_currentRun) { - LOG(warning, "Got a bucket disk move reply while no run is active. " - "This should not happen, as runs should stay active until " - "all requests are answered."); - return true; - } - // Match move against pending ones - Move move; - for (auto it = _pendingMoves.begin(); it != _pendingMoves.end(); ++it) { - if (it->getBucket() == reply->getBucket() - && it->getSourceDisk() == reply->getSrcDisk() - && it->getTargetDisk() == reply->getDstDisk()) - { - move = *it; - _pendingMoves.erase(it); - break; - } - } - // Warn if it wasn't supposed to be active - if (!move.isDefined()) { - LOG(warning, "Got a bucket disk move reply which wasn't registered " - "as pending. This should not happen."); - return true; - } - // Tag move completed in run. - if (reply->getResult().success()) { - _currentRun->moveOk(move); - } else if (reply->getResult().getResult() - == api::ReturnCode::BUCKET_NOT_FOUND - || reply->getResult().getResult() - == api::ReturnCode::BUCKET_DELETED) - { - _currentRun->moveFailedBucketNotFound(move); - } else { - _currentRun->moveFailed(move); - LOGBP(debug, "Failed %s: %s", - move.toString().c_str(), reply->getResult().toString().c_str()); - } - monitor.broadcast(); - return true; -} - -// TODO if we start supporting disk moves for other spaces than the default space -// we also have to check all disk distributions here. -void -BucketMover::storageDistributionChanged() -{ - // Verify that the actual disk distribution changed, if not ignore - lib::Distribution::DiskDistribution newDistr(currentDiskDistribution()); - - if (_diskDistribution == newDistr) { - return; - } - - vespalib::MonitorGuard monitor(_wait); - if (_currentRun) { - LOG(info, "Aborting bucket mover run as disk distribution changed " - "from %s to %s.", - lib::Distribution::getDiskDistributionName(_diskDistribution).c_str(), - lib::Distribution::getDiskDistributionName(newDistr).c_str()); - _currentRun->abort(); - } else { - LOG(info, "Regathering state as disk distribution changed " - "from %s to %s.", - lib::Distribution::getDiskDistributionName(_diskDistribution).c_str(), - lib::Distribution::getDiskDistributionName(newDistr).c_str()); - } - _diskDistribution = newDistr; - _nextRun = framework::SecondTime(0); -} - -lib::Distribution::DiskDistribution BucketMover::currentDiskDistribution() const { - auto distribution = _component.getBucketSpaceRepo().get(document::FixedBucketSpaces::default_space()).getDistribution(); - return distribution->getDiskDistribution(); -} - -bool BucketMover::isWorkingOnCycle() const { - vespalib::MonitorGuard monitor(_wait); - return (_currentRun.get() != nullptr); -} - -uint32_t BucketMover::getCycleCount() const { - vespalib::MonitorGuard monitor(_wait); - return _cycleCount; -} - -void -BucketMover::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - vespalib::MonitorGuard monitor(_wait); - out << "BucketMover() {"; - if (_currentRun) { - out << "\n" << indent << " "; - _currentRun->print(out, verbose, indent + " "); - } else { - out << "\n" << indent << " No current run."; - } - if (verbose && !_history.empty()) { - out << "\n" << indent << " History:"; - for (auto& entry : _history) { - out << "\n" << indent << " "; - entry.print(out, true, indent + " "); - } - } - out << "\n" << indent << "}"; -} - -void -BucketMover::reportHtmlStatus(std::ostream& out, - const framework::HttpUrlPath&) const -{ - vespalib::MonitorGuard monitor(_wait); - if (_history.empty()) { - out << "<h2>Status after last run</h2>\n"; - out << "<p>No run completed yet. Current status unknown.</p>\n"; - } else { - printCurrentStatus(out, *_history.begin()); - } - out << "<h2>Current move cycle</h2>\n"; - if (_currentRun) { - printRunHtml(out, *_currentRun); - if (_currentRun->getPendingMoves().empty()) { - out << "<blockquote>No pending moves.</blockquote>\n"; - } else { - out << "<blockquote>Pending bucket moves:<ul>\n"; - for (auto& entry : _currentRun->getPendingMoves()) { - out << "<li>" << entry << "</li>\n"; - } - out << "</ul></blockquote>\n"; - } - } else { - out << "<p>\n" - << "No bucket move cycle currently running. "; - framework::SecondTime currentTime( - _component.getClock().getTimeInSeconds()); - if (_nextRun <= currentTime) { - if (_thread) { - out << "Next run to start immediately."; - // Wake up thread, so user sees it starts immediately :) - monitor.signal(); - } else { - out << "Waiting for node to finish initialization before " - << "starting run."; - } - } else { - out << "Next run scheduled to run"; - framework::SecondTime diff(_nextRun - currentTime); - if (diff < framework::SecondTime(24 * 60 * 60)) { - out << " in " << diff.toString(framework::DIFFERENCE); - } else { - out << " at time " << _nextRun; - } - out << "."; - } - out << "\n</p>\n"; - } - if (!_history.empty()) { - out << "<h2>Statistics from previous bucket mover cycles</h2>\n"; - for (auto& entry : _history) { - printRunStatisticsHtml(out, entry); - } - } -} - -void -BucketMover::printCurrentStatus(std::ostream& out, - const RunStatistics& rs) const -{ - framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); - out << "<h2>Status after last run (" - << (currentTime - rs._endTime).toString(framework::DIFFERENCE) - << " ago)</h2>\n" - << "<p>Disk distribution: " - << lib::Distribution::getDiskDistributionName(_diskDistribution) - << "</p>\n"; - out << "<p>This is the status from the last completed bucket database scan " - << "done by the bucket mover. After starting storage, or after " - << "configuration changes, a single scan is always done without " - << "actually attempting to move anything, just to get status updated " - << "quickly. During a move cycle, the data shown for the current cycle " - << "will be more recently updated, but will only represent a part of " - << "the bucket database.</p>\n"; - HtmlTable table("Disk"); - table.addColumnHeader("Real partition byte usage", 3); - ByteSizeColumn diskSpaceUsed("Used", &table); - ByteSizeColumn diskSpaceTotal("Total", &table); - DoubleColumn diskSpaceFillRate("Fill rate", " %", &table); - diskSpaceFillRate.addColorLimit(85, Column::LIGHT_GREEN); - diskSpaceFillRate.addColorLimit(95, Column::LIGHT_YELLOW); - diskSpaceFillRate.addColorLimit(100, Column::LIGHT_RED); - diskSpaceFillRate.setTotalAsAverage(); - table.addColumnHeader("Buckets in directory", 2); - LongColumn bucketCount("Count", "", &table); - PercentageColumn bucketCountPart("Part", 0, &table); - table.addColumnHeader("Total document size directory", 2); - ByteSizeColumn documentSize("Size", &table); - PercentageColumn documentSizePart("Part", 0, &table); - table.addColumnHeader("Buckets on correct disk", 2); - LongColumn bucketsCorrectDisk("Count", "", &table); - DoubleColumn bucketsCorrectDiskPart("Part", " %", &table); - bucketsCorrectDiskPart.setTotalAsAverage(); - bucketsCorrectDiskPart.addColorLimit(95, Column::LIGHT_YELLOW); - bucketsCorrectDiskPart.addColorLimit(100, Column::LIGHT_GREEN); - for (uint32_t i=0; i<rs._diskData.size(); ++i) { - table.addRow(i); - // Ignore disks down - bucketCount[i] = rs.getBucketCount(i, true); - bucketCountPart[i] = bucketCount[i]; - documentSize[i] = rs._diskData[i]._bucketSize; - documentSizePart[i] = documentSize[i]; - bucketsCorrectDisk[i] = rs.getBucketCount(i, false); - bucketsCorrectDiskPart[i] = 100.0 * rs.getBucketCount(i, false) - / rs.getBucketCount(i, true); - } - table.addTotalRow("Total"); - table.print(out); - - MATRIX_PRINT("Buckets on wrong disk", _bucketsLeftOnWrongDisk, rs); -} - -void -BucketMover::printRunHtml(std::ostream& out, const bucketmover::Run& runner) const -{ - printRunStatisticsHtml(out, runner.getStatistics()); -} - -void -BucketMover::printRunStatisticsHtml(std::ostream& out, - const RunStatistics& rs) const -{ - rs.print(out, true, ""); -} - -} diff --git a/storage/src/vespa/storage/bucketmover/bucketmover.h b/storage/src/vespa/storage/bucketmover/bucketmover.h deleted file mode 100644 index 57829eb91bd..00000000000 --- a/storage/src/vespa/storage/bucketmover/bucketmover.h +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::BucketMover - * @ingroup storageserver - * - * @brief This class moves buckets between disks for reducing node skew. Highly - * inspired from BucketIntegrityChecker. - * - * It uses DiskMonitor class to monitor disk info (space available, space used, - * etc), but also to monitor the number of pending moves for each disk. - * It also uses BucketMoverHeuristic class to decide on which buckets should be - * moved and to what disk. - * - * @version $Id: - */ - -#pragma once - -#include "run.h" -#include <vespa/storage/common/storagelink.h> -#include <vespa/storage/config/config-stor-bucketmover.h> -#include <vespa/storage/common/servicelayercomponent.h> -#include <vespa/storageframework/generic/status/htmlstatusreporter.h> -#include <vespa/storageapi/message/bucket.h> -#include <vespa/vdslib/distribution/distribution.h> -#include <vespa/config/helper/ifetchercallback.h> -#include <vespa/config/subscription/configuri.h> -#include <vespa/config/config.h> - -namespace storage { - -class BucketDiskMoveCommand; - -namespace bucketmover { - -class BucketMover : public StorageLink, - private framework::Runnable, - public framework::HtmlStatusReporter, - private config::IFetcherCallback<vespa::config::content::core::StorBucketmoverConfig> -{ - ServiceLayerComponent _component; - std::unique_ptr<vespa::config::content::core::StorBucketmoverConfig> _config; - uint32_t _cycleCount; - framework::SecondTime _nextRun; - std::unique_ptr<bucketmover::Run> _currentRun; - std::list<Move> _pendingMoves; - std::list<std::shared_ptr<BucketDiskMoveCommand> > _newMoves; - std::list<RunStatistics> _history; - vespalib::Monitor _wait; - config::ConfigFetcher _configFetcher; - lib::Distribution::DiskDistribution _diskDistribution; - uint32_t _maxSleepTime; - framework::Thread::UP _thread; - -public: - BucketMover(const config::ConfigUri & configUri, ServiceLayerComponentRegister&); - ~BucketMover(); - - void onDoneInit() override; - void onClose() override; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - bool isWorkingOnCycle() const; - uint32_t getCycleCount() const; - void signal(); - framework::SecondTime getNextRunTime() const { return _nextRun; } - - // Useful for unit testing - vespa::config::content::core::StorBucketmoverConfig& getConfig() { return *_config; } - RunStatistics& getLastRunStats() { return *_history.begin(); } - bool tick(); - void finishCurrentRun(); - -private: - friend struct BucketMoverTest; - - void startNewRun(); - void queueNewMoves(); - void sendNewMoves(); - - void configure(std::unique_ptr<vespa::config::content::core::StorBucketmoverConfig>) override; - void run(framework::ThreadHandle&) override; - bool onInternalReply(const std::shared_ptr<api::InternalReply>&) override; - void storageDistributionChanged() override; - lib::Distribution::DiskDistribution currentDiskDistribution() const; - - framework::SecondTime calculateWaitTimeOfNextRun() const; - - void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; - void printCurrentStatus(std::ostream&, const RunStatistics&) const; - void printRunHtml(std::ostream&, const bucketmover::Run&) const; - void printRunStatisticsHtml(std::ostream&, const RunStatistics&) const; -}; - -} // bucketmover -} // storage diff --git a/storage/src/vespa/storage/bucketmover/move.cpp b/storage/src/vespa/storage/bucketmover/move.cpp deleted file mode 100644 index 12b38980818..00000000000 --- a/storage/src/vespa/storage/bucketmover/move.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "move.h" -#include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <ostream> - -namespace storage { -namespace bucketmover { - -Move::Move() - : _sourceDisk(0), - _targetDisk(0), - _bucket(), - _totalDocSize(0), - _priority(255) -{ -} - -Move::Move(uint16_t source, uint16_t target, const document::Bucket& bucket, - uint32_t totalDocSize) - : _sourceDisk(source), - _targetDisk(target), - _bucket(bucket), - _totalDocSize(totalDocSize), - _priority(255) -{ -} - -void -Move::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - (void) verbose; (void) indent; - if (!isDefined()) { - out << "Move(none)"; - return; - } - out << "Move(" << _bucket << ", " << _sourceDisk << " -> " << _targetDisk - << ", pri " << (uint16_t) _priority << ")"; -} - -} // bucketmover -} // storage diff --git a/storage/src/vespa/storage/bucketmover/move.h b/storage/src/vespa/storage/bucketmover/move.h deleted file mode 100644 index 92d05e4e0ae..00000000000 --- a/storage/src/vespa/storage/bucketmover/move.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::bucketmover::Move - * \ingroup bucketmover - * - * \brief Class representing a bucket move between disks. - */ - -#pragma once - -#include <vespa/document/bucket/bucket.h> -#include <vespa/vespalib/util/printable.h> - -namespace storage { -namespace bucketmover { - -class Move : public vespalib::Printable { - uint16_t _sourceDisk; - uint16_t _targetDisk; - document::Bucket _bucket; - uint32_t _totalDocSize; - uint8_t _priority; - -public: - Move(); - Move(uint16_t source, uint16_t target, const document::Bucket& bucket, - uint32_t totalDocSize); - - /** False if invalid move. (Empty constructor) Indicates end of run. */ - bool isDefined() const { return (_bucket.getBucketId().getRawId() != 0); } - - // Only valid to call if move is defined - uint16_t getSourceDisk() const { return _sourceDisk; } - uint16_t getTargetDisk() const { return _targetDisk; } - const document::Bucket& getBucket() const { return _bucket; } - uint8_t getPriority() const { return _priority; } - uint32_t getTotalDocSize() const { return _totalDocSize; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} // bucketmover -} // storage - diff --git a/storage/src/vespa/storage/bucketmover/run.cpp b/storage/src/vespa/storage/bucketmover/run.cpp deleted file mode 100644 index 22bcfa55f15..00000000000 --- a/storage/src/vespa/storage/bucketmover/run.cpp +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "run.h" -#include <vespa/storage/bucketdb/storbucketdb.h> -#include <vespa/storage/bucketdb/lockablemap.hpp> -#include <vespa/storageframework/generic/clock/clock.h> -#include <iomanip> - -#include <vespa/log/log.h> -LOG_SETUP(".bucketmover.run"); - -namespace storage { -namespace bucketmover { - -Run::Run(ContentBucketSpace& bucketSpace, - const lib::NodeState& nodeState, - uint16_t nodeIndex, - framework::Clock& clock) - : _bucketSpace(bucketSpace), - _distribution(bucketSpace.getDistribution()), - _nodeState(nodeState), - _nodeIndex(nodeIndex), - _entries(), - _iterationDone(false), - _statistics(_distribution->getDiskDistribution(), clock, nodeState), - _aborted(false) -{ -} - -namespace { - struct BucketIterator { - document::BucketSpace _iteratedBucketSpace; - const lib::Distribution& _distribution; - const lib::NodeState& _nodeState; - RunStatistics& _statistics; - std::list<Move>& _entries; - uint16_t _nodeIndex; - uint32_t _maxBucketsToIterateAtOnce; - uint32_t _bucketsVisited; - document::BucketId _firstBucket; - - BucketIterator(document::BucketSpace iteratedBucketSpace, - const lib::Distribution& d, const lib::NodeState& ns, - uint16_t nodeIndex, RunStatistics& stats, - std::list<Move>& entries) - : _iteratedBucketSpace(iteratedBucketSpace), - _distribution(d), - _nodeState(ns), - _statistics(stats), - _entries(entries), - _nodeIndex(nodeIndex), - _maxBucketsToIterateAtOnce(10000), - _bucketsVisited(0), - _firstBucket(stats._lastBucketVisited) - { - } - - StorBucketDatabase::Decision - operator()(document::BucketId::Type revId, - StorBucketDatabase::Entry& entry) - { - document::BucketId bucketId(document::BucketId::keyToBucketId(revId)); - if (bucketId == _firstBucket) { - return StorBucketDatabase::CONTINUE; - } - uint16_t idealDisk = _distribution.getIdealDisk( - _nodeState, _nodeIndex, bucketId, - lib::Distribution::IDEAL_DISK_EVEN_IF_DOWN); - RunStatistics::DiskData& diskData( - _statistics._diskData[entry.disk]); - bool idealDiskDown( - _statistics._diskData[idealDisk]._diskDisabled); - if (entry.disk == idealDisk || idealDiskDown) { - diskData._bucketSize += entry.getBucketInfo().getTotalDocumentSize(); - ++diskData._bucketsFoundOnCorrectDisk; - } else { - document::Bucket bucket(_iteratedBucketSpace, bucketId); - _entries.push_back(Move( - entry.disk, idealDisk, bucket, entry.getBucketInfo().getTotalDocumentSize())); - } - _statistics._lastBucketVisited = bucketId; - if (++_bucketsVisited >= _maxBucketsToIterateAtOnce) { - return StorBucketDatabase::ABORT; - } - return StorBucketDatabase::CONTINUE; - } - }; -} - -Move -Run::getNextMove() -{ - if (_aborted) { - LOG(debug, "Run aborted. Returning undefined move."); - return Move(); - } - if (_iterationDone) { - LOG(debug, "Run completed. End time set. Returning undefined move."); - return Move(); - } - while (true) { - // Process cached entries until we either found one to move, or - // we have no more - while (!_entries.empty()) { - Move e(_entries.front()); - _entries.pop_front(); - - if (!_statistics._diskData[e.getTargetDisk()]._diskDisabled) { - _pending.push_back(e); - _statistics._lastBucketProcessed = e.getBucket(); // Only used for printing - _statistics._lastBucketProcessedTime = _statistics._clock->getTimeInSeconds(); - return e; - } - } - - // Cache more entries - BucketIterator it(_bucketSpace.bucketSpace(), *_distribution, - _nodeState, _nodeIndex, _statistics, _entries); - _bucketSpace.bucketDatabase().all(it, "bucketmover::Run", _statistics._lastBucketVisited.toKey()); - if (it._bucketsVisited == 0) { - _iterationDone = true; - if (_pending.empty()) { - finalize(); - } - LOG(debug, "Last bucket visited. Done iterating buckets in run."); - return Move(); - } - } -} - -void -Run::finalize() -{ - _statistics._endTime = _statistics._clock->getTimeInSeconds(); -} - -void -Run::removePending(Move& move) -{ - bool foundPending = false; - for (auto it = _pending.begin(); it != _pending.end(); ++it) { - if (it->getBucket() == move.getBucket()) { - _pending.erase(it); - foundPending = true; - break; - } - } - if (!foundPending) { - LOG(warning, "Got answer for %s that was not in the pending list.", - move.getBucket().toString().c_str()); - return; - } - if (_iterationDone && _pending.empty()) { - finalize(); - } -} - -void -Run::moveOk(Move& move) -{ - ++_statistics._diskData[move.getSourceDisk()][move.getTargetDisk()] - ._bucketsMoved; - removePending(move); - uint32_t size = move.getTotalDocSize(); - - _statistics._diskData[move.getSourceDisk()]._bucketSize -= size; - _statistics._diskData[move.getTargetDisk()]._bucketSize += size; -} - -void -Run::moveFailedBucketNotFound(Move& move) -{ - ++_statistics._diskData[move.getSourceDisk()][move.getTargetDisk()] - ._bucketsNotFoundAtExecutionTime; - removePending(move); -} - -void -Run::moveFailed(Move& move) -{ - ++_statistics._diskData[move.getSourceDisk()][move.getTargetDisk()] - ._bucketsFailedMoving; - _statistics._diskData[move.getTargetDisk()]._diskDisabled = true; - removePending(move); -} - -void -Run::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "Run("; - if (_aborted) { - out << "Aborted"; - } else if (_statistics._endTime.isSet()) { - if (_entries.empty()) { - out << "Completed"; - } else { - out << "Iteration done"; - } - } - out << ") {\n" << indent << " "; - _statistics.print(out, verbose, indent + " "); - if (!_entries.empty()) { - out << "\n" << indent << " Pending possible moves:"; - uint32_t i = 0; - for (std::list<Move>::const_iterator it = _entries.begin(); - it != _entries.end() && ++i <= 10; ++it) - { - out << "\n" << indent << " " << *it; - } - uint32_t size = _entries.size(); - if (size > 10) { - out << "\n" << indent << " ... and " << (size - 10) - << " more."; - } - } - if (!_statistics._endTime.isSet()) { - out << "\n" << indent << " Bucket iterator: " - << _statistics._lastBucketVisited; - } - out << "\n" << indent << "}"; -} - -} // bucketmover -} // storage diff --git a/storage/src/vespa/storage/bucketmover/run.h b/storage/src/vespa/storage/bucketmover/run.h deleted file mode 100644 index 914781eec30..00000000000 --- a/storage/src/vespa/storage/bucketmover/run.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::bucketmover::Run - * \ingroup storageserver - * - * \brief The run class takes care of creating operations for a single run. - * - * The run class keeps a matrix that keeps track of up to a max number of - * buckets that are located on the wrong disk. The first index of the matrix is - * the source disk and the second index is the ideal disk. - * - * The Run does not care about about pending or wait for that. The caller of - * getNextMove() have to regulate max pending for themselevs. - */ - -#pragma once - - -#include "move.h" -#include "runstatistics.h" -#include <vespa/storage/common/content_bucket_space.h> -#include <vespa/vdslib/distribution/distribution.h> -#include <vespa/vdslib/state/nodestate.h> -#include <list> -#include <map> - -namespace storage { - -class MountPointList; -class PartitionMonitor; -class StorBucketDatabase; -class Clock; - -namespace bucketmover { - -class Run : public document::Printable { - ContentBucketSpace& _bucketSpace; - std::shared_ptr<const lib::Distribution> _distribution; - lib::NodeState _nodeState; - uint16_t _nodeIndex; - std::list<Move> _entries; - std::list<Move> _pending; - bool _iterationDone; - RunStatistics _statistics; - bool _aborted; - std::map<uint16_t, bool> _diskDisabled; - -public: - Run(const Run &) = delete; - Run & operator = (const Run &) = delete; - Run(ContentBucketSpace& bucketSpace, - const lib::NodeState&, - uint16_t nodeIndex, - framework::Clock&); - - /** - * If disk distribution change during runs, they get aborted. We want to - * track this in run, as we want run to exist until all pending requests - * have been answered. - */ - void abort() { _aborted = true; } - bool aborted() { return _aborted; } - - /** - * Get next move does the following: - * - Sort disks in order of fillrate. (PartitionMonitor keeps cache, so - * this will only stat once in a while) - * - If the matrix contains a possible move from above average fill rate - * to below average fill rate, do that move. Prioritizing moving away - * from fullest disk. - * - Otherwise, continue visiting bucket database to fill up matrix. - * - If any moves left, do next, prioritizing moving away from fullest - * disk. - * - * @return A Move object. If isDefined() returns false, run is complete. - * The whole database have been iterated through. - */ - Move getNextMove(); - - void moveOk(Move& move); - void moveFailedBucketNotFound(Move& move); - void moveFailed(Move& move); - - const std::list<Move>& getPendingMoves() const { return _pending; } - - RunStatistics& getStatistics() { return _statistics; } - const RunStatistics& getStatistics() const { return _statistics; } - void print(std::ostream&, bool verbose, const std::string& indent) const override; -private: - void removePending(Move&); - void finalize(); -}; - -} // bucketmover -} // storage diff --git a/storage/src/vespa/storage/bucketmover/runstatistics.cpp b/storage/src/vespa/storage/bucketmover/runstatistics.cpp deleted file mode 100644 index 314f04a0d66..00000000000 --- a/storage/src/vespa/storage/bucketmover/runstatistics.cpp +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "runstatistics.h" -#include "htmltable.h" -#include <vespa/storageframework/generic/clock/clock.h> - - -namespace storage::bucketmover { - -RunStatistics::DiskMatrix::DiskMatrix() - : _bucketsMoved(0), - _bucketsFailedMoving(0), - _bucketsLeftOnWrongDisk(0), - _bucketsNotFoundAtExecutionTime(0) -{ -} - -RunStatistics::DiskData::DiskData(uint16_t diskCount) - : _targetDisks(diskCount), - _bucketsFoundOnCorrectDisk(0), - _bucketSize(0), - _diskDisabled(false) -{ -} - -double -RunStatistics::DiskData::getWronglyPlacedRatio() const -{ - uint64_t wrong = 0; - for (uint32_t i=0; i<_targetDisks.size(); ++i) { - wrong += _targetDisks[i]._bucketsLeftOnWrongDisk - + _targetDisks[i]._bucketsFailedMoving; - } - uint64_t total = wrong + _bucketsFoundOnCorrectDisk; - return static_cast<double>(wrong) / total; -} - -RunStatistics::RunStatistics(DiskDistribution d, framework::Clock& clock, - const lib::NodeState& ns) - : _clock(&clock), - _distribution(d), - _lastBucketProcessed(), - _lastBucketVisited(0), - _diskData(ns.getDiskCount(), DiskData(ns.getDiskCount())), - _startTime(_clock->getTimeInSeconds()), - _endTime(0), - _lastBucketProcessedTime(0) -{ - for (uint32_t i=0; i<ns.getDiskCount(); ++i) { - if (!ns.getDiskState(i).getState().oneOf("uis")) { - _diskData[i]._diskDisabled = true; - } - } -} - -void -RunStatistics::print(std::ostream& out, bool verbose, - const std::string& ind) const -{ - (void) verbose; (void) ind; - bool completed(_endTime.isSet()); - framework::SecondTime currentTime = _clock->getTimeInSeconds(); - if (completed) { - out << "<h3>Run from " << _startTime << " to " << _endTime; - } else { - out << "<h3>Run started " - << currentTime.getDiff(_startTime).toString(framework::DIFFERENCE) - << " ago"; - } - out << " with distribution " - << lib::Distribution::getDiskDistributionName(_distribution) - << "</h3>\n<blockquote>"; - if (!completed) { - std::ostringstream progress; - progress << std::fixed << std::setprecision(4) - << (100.0 * getProgress()); - out << "<p>Progress: " << progress.str() << " % "; - if (_lastBucketProcessedTime.isSet()) { - out << "<font color=\"gray\" size=\"-1\">Last move for " - << _lastBucketProcessed << " " - << currentTime.getDiff(_lastBucketProcessedTime) - .toString(framework::DIFFERENCE) - << " ago</font>"; - } - out << "</p>\n"; - } - - HtmlTable table("Disk"); - table.addColumnHeader(completed ? "Buckets in directory after run" - : "Processed buckets in directory", 2); - LongColumn bucketCount("Count", "", &table); - PercentageColumn bucketCountPart("Part", 0, &table); - - table.addColumnHeader(completed - ? "Total document size in directory after run" - : "Total document size of processed buckets in directory", 2); - ByteSizeColumn documentSize("Size", &table); - PercentageColumn documentSizePart("Part", 0, &table); - - table.addColumnHeader(completed ? "Buckets on correct disk after run" - : "Processed buckets on correct disk", 2); - LongColumn bucketsCorrectDisk("Count", "", &table); - DoubleColumn bucketsCorrectDiskPart("Part", " %", &table); - bucketsCorrectDiskPart.setTotalAsAverage(); - bucketsCorrectDiskPart.addColorLimit(95, Column::LIGHT_YELLOW); - bucketsCorrectDiskPart.addColorLimit(100, Column::LIGHT_GREEN); - - for (uint32_t i=0; i<_diskData.size(); ++i) { - table.addRow(i); - if (_diskData[i]._diskDisabled) { - table.setRowHeaderColor(Column::LIGHT_RED); - } - - bucketCount[i] = getBucketCount(i, true); - bucketCountPart[i] = bucketCount[i]; - - documentSize[i] = _diskData[i]._bucketSize; - documentSizePart[i] = documentSize[i]; - - bucketsCorrectDisk[i] = getBucketCount(i, false); - bucketsCorrectDiskPart[i] - = 100.0 * getBucketCount(i, false) / getBucketCount(i, true); - } - table.addTotalRow("Total"); - table.print(out); - - MATRIX_PRINT("Buckets left on wrong disk", _bucketsLeftOnWrongDisk, *this); - MATRIX_PRINT("Buckets moved", _bucketsMoved, *this); - MATRIX_PRINT("Buckets not found at move time", - _bucketsNotFoundAtExecutionTime, *this); - MATRIX_PRINT("Buckets failed moving for other reasons", - _bucketsFailedMoving, *this); - - out << "</blockquote>\n"; -} - -double -RunStatistics::getWronglyPlacedRatio() const -{ - uint64_t wrong = 0, total = 0; - for (uint32_t i=0; i<_diskData.size(); ++i) { - for (uint32_t j=0; j<_diskData.size(); ++j) { - wrong += _diskData[i][j]._bucketsLeftOnWrongDisk - + _diskData[i][j]._bucketsFailedMoving; - } - total += _diskData[i]._bucketsFoundOnCorrectDisk; - } - total += wrong; - return static_cast<double>(wrong) / total; -} - -// FIXME does not cover multiple spaces (but only used for printing) -double -RunStatistics::getProgress() const -{ - if (_endTime.isSet()) return 1.0; - double result = 0; - double weight = 0.5; - uint64_t key = _lastBucketProcessed.getBucketId().toKey(); - for (uint16_t i=0; i<64; ++i) { - uint64_t flag = uint64_t(1) << (63 - i); - if ((key & flag) == flag) { - result += weight; - } - weight /= 2; - } - return result; -} - -uint64_t -RunStatistics::getBucketCount(uint16_t disk, bool includeWrongLocation) const -{ - uint64_t total = 0; - for (uint32_t i=0; i<_diskData.size(); ++i) { - if (disk == i) total += _diskData[i]._bucketsFoundOnCorrectDisk; - for (uint32_t j=0; j<_diskData.size(); ++j) { - if (disk == i) { - if (includeWrongLocation) { - total += _diskData[i][j]._bucketsLeftOnWrongDisk; - total += _diskData[i][j]._bucketsFailedMoving; - } - } else if (disk == j) { - total += _diskData[i][j]._bucketsMoved; - } - } - } - return total; -} - -} diff --git a/storage/src/vespa/storage/bucketmover/runstatistics.h b/storage/src/vespa/storage/bucketmover/runstatistics.h deleted file mode 100644 index 908f345b307..00000000000 --- a/storage/src/vespa/storage/bucketmover/runstatistics.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::bucketmover::RunStatistics - * \ingroup bucketmover - * - * \brief Statistics gathered from a bucket mover cycle. - */ - -#pragma once - -#define MATRIX_PRINT(desc, var, rs) \ -{ \ - bool anyset = false; \ - for (uint32_t i=0; i<(rs)._diskData.size(); ++i) { \ - for (uint32_t j=0; j<(rs)._diskData.size(); ++j) { \ - anyset |= ((rs)._diskData[i][j].var > 0); \ - } \ - } \ - if (anyset) { \ - out << "<h4>" << desc << "</h4>\n"; \ - HtmlTable matrixTable("Source \\ Target"); \ - using LCUP = std::unique_ptr<LongColumn>; \ - std::vector<LCUP> matrixData((rs)._diskData.size()); \ - for (uint32_t i=0; i<(rs)._diskData.size(); ++i) { \ - std::ostringstream index; \ - index << "Disk " << i; \ - matrixData[i].reset(new LongColumn(index.str(), "", &matrixTable));\ - matrixTable.addRow(index.str()); \ - } \ - for (uint32_t i=0; i<(rs)._diskData.size(); ++i) { \ - for (uint32_t j=0; j<(rs)._diskData.size(); ++j) { \ - (*matrixData[j])[i] = (rs)._diskData[i][j].var; \ - } \ - } \ - matrixTable.print(out); \ - } \ -} - -#include <vespa/vdslib/state/nodestate.h> -#include <vespa/vdslib/distribution/distribution.h> -#include <vespa/document/bucket/bucket.h> -#include <vespa/vespalib/util/printable.h> -#include <vespa/storageframework/generic/clock/time.h> - -#include <vector> - -namespace storage::bucketmover { - -struct RunStatistics : public document::Printable { - using DiskDistribution = lib::Distribution::DiskDistribution; - - /** Data kept as targets for moves for each disk. */ - struct DiskMatrix { - uint32_t _bucketsMoved; - uint32_t _bucketsFailedMoving; - uint32_t _bucketsLeftOnWrongDisk; - uint32_t _bucketsNotFoundAtExecutionTime; - - DiskMatrix(); - }; - - /** Data kept per disk. */ - struct DiskData { - std::vector<DiskMatrix> _targetDisks; - uint32_t _bucketsFoundOnCorrectDisk; - uint64_t _bucketSize; - bool _diskDisabled; - - DiskData(uint16_t diskCount); - - DiskMatrix& operator[](uint16_t index) { return _targetDisks[index]; } - const DiskMatrix& operator[](uint16_t index) const - { return _targetDisks[index]; } - double getWronglyPlacedRatio() const; - }; - - framework::Clock* _clock; - DiskDistribution _distribution; - document::Bucket _lastBucketProcessed; - document::BucketId _lastBucketVisited; // Invalid bucket for starting point - std::vector<DiskData> _diskData; - framework::SecondTime _startTime; - framework::SecondTime _endTime; - framework::SecondTime _lastBucketProcessedTime; - - RunStatistics(DiskDistribution, framework::Clock&, const lib::NodeState&); - - double getWronglyPlacedRatio() const; - double getProgress() const; - uint64_t getBucketCount(uint16_t disk, bool includeWrongLocation) const; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} diff --git a/storage/src/vespa/storage/common/bucketmessages.cpp b/storage/src/vespa/storage/common/bucketmessages.cpp index f953428fc8a..d7f75f12013 100644 --- a/storage/src/vespa/storage/common/bucketmessages.cpp +++ b/storage/src/vespa/storage/common/bucketmessages.cpp @@ -106,60 +106,6 @@ std::unique_ptr<api::StorageReply> ReadBucketInfo::makeReply() { return std::make_unique<ReadBucketInfoReply>(*this); } -BucketDiskMoveCommand::BucketDiskMoveCommand(const document::Bucket &bucket, - uint16_t srcDisk, uint16_t dstDisk) - : api::InternalCommand(ID), - _bucket(bucket), - _srcDisk(srcDisk), - _dstDisk(dstDisk) -{ - setPriority(LOW); -} - -BucketDiskMoveCommand::~BucketDiskMoveCommand() = default; - -void -BucketDiskMoveCommand::setBucketId(const document::BucketId& id) -{ - document::Bucket newBucket(_bucket.getBucketSpace(), id); - _bucket = newBucket; -} - -void -BucketDiskMoveCommand::print(std::ostream& out, bool, const std::string&) const { - out << "BucketDiskMoveCommand(" << _bucket.getBucketId() << ", source " << _srcDisk - << ", target " << _dstDisk << ")"; -} - -BucketDiskMoveReply::BucketDiskMoveReply(const BucketDiskMoveCommand& cmd, - const api::BucketInfo& bucketInfo, - uint32_t sourceFileSize, - uint32_t destinationFileSize) - : api::InternalReply(ID, cmd), - _bucket(cmd.getBucket()), - _bucketInfo(bucketInfo), - _fileSizeOnSrc(sourceFileSize), - _fileSizeOnDst(destinationFileSize), - _srcDisk(cmd.getSrcDisk()), - _dstDisk(cmd.getDstDisk()) -{ } - -BucketDiskMoveReply::~BucketDiskMoveReply() = default; - -void -BucketDiskMoveReply::print(std::ostream& out, bool, const std::string&) const -{ - out << "BucketDiskMoveReply(" << _bucket.getBucketId() << ", source " << _srcDisk - << ", target " << _dstDisk << ", " << _bucketInfo << ", " << getResult() << ")"; -} - -std::unique_ptr<api::StorageReply> -BucketDiskMoveCommand::makeReply() -{ - return std::make_unique<BucketDiskMoveReply>(*this); -} - - InternalBucketJoinCommand::InternalBucketJoinCommand(const document::Bucket &bucket, uint16_t keepOnDisk, uint16_t joinFromDisk) : api::InternalCommand(ID), diff --git a/storage/src/vespa/storage/common/bucketmessages.h b/storage/src/vespa/storage/common/bucketmessages.h index fed944fb217..d6ba7915b3f 100644 --- a/storage/src/vespa/storage/common/bucketmessages.h +++ b/storage/src/vespa/storage/common/bucketmessages.h @@ -115,76 +115,6 @@ public: }; /** - * @class BucketDiskMoveCommand - * @ingroup common - * - * @brief Move a given bucket (from src disk to dst disk). - * - * This message is sent continually by the bucket mover. - * Size of the bucket moved is reported back. - */ -class BucketDiskMoveCommand : public api::InternalCommand { - document::Bucket _bucket; - uint16_t _srcDisk; - uint16_t _dstDisk; - -public: - typedef std::shared_ptr<BucketDiskMoveCommand> SP; - static const uint32_t ID = 2012; - - BucketDiskMoveCommand(const document::Bucket &bucket, uint16_t srcDisk, uint16_t dstDisk); - ~BucketDiskMoveCommand(); - - document::Bucket getBucket() const override { return _bucket; } - bool hasSingleBucketId() const override { return true; } - - uint16_t getSrcDisk() const { return _srcDisk; } - uint16_t getDstDisk() const { return _dstDisk; } - - void setBucketId(const document::BucketId& id); - - std::unique_ptr<api::StorageReply> makeReply() override; - - void print(std::ostream& out, bool, const std::string&) const override; -}; - -/** - * @class BucketDiskMoveReply - * @ingroup common - */ -class BucketDiskMoveReply : public api::InternalReply { - document::Bucket _bucket; - api::BucketInfo _bucketInfo; - uint64_t _fileSizeOnSrc; - uint64_t _fileSizeOnDst; - uint16_t _srcDisk; - uint16_t _dstDisk; - -public: - typedef std::shared_ptr<BucketDiskMoveReply> SP; - static const uint32_t ID = 2013; - - BucketDiskMoveReply(const BucketDiskMoveCommand& cmd, - const api::BucketInfo& bucketInfo = api::BucketInfo(), - uint32_t sourceFileSize = 0, - uint32_t destinationFileSize = 0); - ~BucketDiskMoveReply(); - - document::Bucket getBucket() const override { return _bucket; } - bool hasSingleBucketId() const override { return true; } - - const api::BucketInfo& getBucketInfo() const { return _bucketInfo; } - void setFileSizeOnSrc(uint64_t fileSize) { _fileSizeOnSrc = fileSize; } - void setFileSizeOnDst(uint64_t fileSize) { _fileSizeOnDst = fileSize; } - uint64_t getFileSizeOnSrc() const { return _fileSizeOnSrc; } - uint64_t getFileSizeOnDst() const { return _fileSizeOnDst; } - uint16_t getSrcDisk() const { return _srcDisk; } - uint16_t getDstDisk() const { return _dstDisk; } - - void print(std::ostream& out, bool, const std::string&) const override; -}; - -/** * @class InternalBucketJoinCommand * @ingroup common * diff --git a/storage/src/vespa/storage/common/messagebucket.cpp b/storage/src/vespa/storage/common/messagebucket.cpp index 73c0827c05b..6cc94e2a732 100644 --- a/storage/src/vespa/storage/common/messagebucket.cpp +++ b/storage/src/vespa/storage/common/messagebucket.cpp @@ -64,8 +64,6 @@ getStorageMessageBucket(const api::StorageMessage& msg) return static_cast<const ReadBucketList&>(msg).getBucket(); case ReadBucketInfo::ID: return static_cast<const ReadBucketInfo&>(msg).getBucket(); - case BucketDiskMoveCommand::ID: - return static_cast<const BucketDiskMoveCommand&>(msg).getBucket(); case InternalBucketJoinCommand::ID: return static_cast<const InternalBucketJoinCommand&>(msg).getBucket(); case RecheckBucketInfoCommand::ID: diff --git a/storage/src/vespa/storage/frameworkimpl/thread/CMakeLists.txt b/storage/src/vespa/storage/frameworkimpl/thread/CMakeLists.txt index 603ddf26b90..6cc60e53780 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/CMakeLists.txt +++ b/storage/src/vespa/storage/frameworkimpl/thread/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(storage_thread OBJECT SOURCES appkiller.cpp deadlockdetector.cpp + htmltable.cpp DEPENDS AFTER storage_storageconfig diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp index e1b7d9c52ea..fad99e4e5d3 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "deadlockdetector.h" +#include "htmltable.h" #include <vespa/storage/bucketdb/storbucketdb.h> -#include <vespa/storage/bucketmover/htmltable.h> #include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/storage/src/vespa/storage/bucketmover/htmltable.cpp b/storage/src/vespa/storage/frameworkimpl/thread/htmltable.cpp index 7596e221092..7596e221092 100644 --- a/storage/src/vespa/storage/bucketmover/htmltable.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/htmltable.cpp diff --git a/storage/src/vespa/storage/bucketmover/htmltable.h b/storage/src/vespa/storage/frameworkimpl/thread/htmltable.h index 85e9438edcb..85e9438edcb 100644 --- a/storage/src/vespa/storage/bucketmover/htmltable.h +++ b/storage/src/vespa/storage/frameworkimpl/thread/htmltable.h diff --git a/storage/src/vespa/storage/persistence/CMakeLists.txt b/storage/src/vespa/storage/persistence/CMakeLists.txt index 9edc3bc0080..08e98e9a58e 100644 --- a/storage/src/vespa/storage/persistence/CMakeLists.txt +++ b/storage/src/vespa/storage/persistence/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_library(storage_spersistence OBJECT SOURCES bucketownershipnotifier.cpp bucketprocessor.cpp - diskmoveoperationhandler.cpp fieldvisitor.cpp mergehandler.cpp messages.cpp diff --git a/storage/src/vespa/storage/persistence/diskmoveoperationhandler.cpp b/storage/src/vespa/storage/persistence/diskmoveoperationhandler.cpp deleted file mode 100644 index 18877bdf8f7..00000000000 --- a/storage/src/vespa/storage/persistence/diskmoveoperationhandler.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "diskmoveoperationhandler.h" - -#include <vespa/log/log.h> -LOG_SETUP(".persistence.diskmoveoperationhandler"); - -namespace storage { - -DiskMoveOperationHandler::DiskMoveOperationHandler(PersistenceUtil& env, spi::PersistenceProvider& provider) - : _env(env), - _provider(provider) -{ -} - -MessageTracker::UP -DiskMoveOperationHandler::handleBucketDiskMove(BucketDiskMoveCommand& cmd, MessageTracker::UP tracker) -{ - tracker->setMetric(_env._metrics.movedBuckets); - - document::Bucket bucket(cmd.getBucket()); - uint32_t targetDisk(cmd.getDstDisk()); - uint32_t deviceIndex(_env._partition); - - if (cmd.getSrcDisk() != deviceIndex) { - tracker->fail(api::ReturnCode::INTERNAL_FAILURE, - "Tried to move bucket from source disk where it was not located"); - return tracker; - } - if (targetDisk == deviceIndex) { - tracker->fail(api::ReturnCode::INTERNAL_FAILURE, - "Tried to move bucket from and to the same disk"); - return tracker; - } - if (!_env._fileStorHandler.enabled(targetDisk)) { - tracker->fail(api::ReturnCode::ABORTED, "Target disk is not available"); - return tracker; - } - - LOG(debug, "Moving bucket %s from disk %u to disk %u.", - bucket.toString().c_str(), - deviceIndex, targetDisk); - - spi::Bucket from(bucket, spi::PartitionId(deviceIndex)); - - spi::Result result(_provider.move(from, spi::PartitionId(targetDisk), tracker->context())); - if (result.hasError()) { - tracker->fail(api::ReturnCode::INTERNAL_FAILURE, result.getErrorMessage()); - return tracker; - } - - api::BucketInfo bInfo = _env.getBucketInfo(bucket, targetDisk); - uint32_t sourceFileSize = bInfo.getUsedFileSize(); - - { - // Grab bucket lock in bucket database, and update it - // If entry doesn't exist, that means it has just been deleted by - // delete bucket command. If so, it'll be deleted when delete bucket - // is executed. moving queue should move delete command to correct disk - StorBucketDatabase::WrappedEntry entry( - _env.getBucketDatabase(bucket.getBucketSpace()).get( - bucket.getBucketId(), "FileStorThread::onBucketDiskMove", - StorBucketDatabase::LOCK_IF_NONEXISTING_AND_NOT_CREATING)); - - // Move queued operations in bucket to new thread. Hold bucket lock - // while doing it, so filestor manager can't put in other operations - // first, such that operations change order. - _env._fileStorHandler.remapQueueAfterDiskMove(bucket, deviceIndex, targetDisk); - - if (entry.exist()) { - entry->setBucketInfo(bInfo); - entry->disk = targetDisk; - entry.write(); - } - } - - // Answer message, setting extra info such as filesize - tracker->setReply(std::make_shared<BucketDiskMoveReply>(cmd, bInfo, sourceFileSize, sourceFileSize)); - - return tracker; -} - -} // storage diff --git a/storage/src/vespa/storage/persistence/diskmoveoperationhandler.h b/storage/src/vespa/storage/persistence/diskmoveoperationhandler.h deleted file mode 100644 index 9e8d33fc802..00000000000 --- a/storage/src/vespa/storage/persistence/diskmoveoperationhandler.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/storage/common/bucketmessages.h> -#include <vespa/storage/persistence/persistenceutil.h> - -namespace storage { - -class DiskMoveOperationHandler : public Types { - -public: - DiskMoveOperationHandler(PersistenceUtil&, - spi::PersistenceProvider& provider); - - MessageTracker::UP handleBucketDiskMove(BucketDiskMoveCommand&, MessageTracker::UP tracker); - -private: - PersistenceUtil& _env; - spi::PersistenceProvider& _provider; -}; - -} // storage - diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index e23d65f72ac..5627ade2a11 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -658,22 +658,6 @@ FileStorHandlerImpl::remapMessage(api::StorageMessage& msg, const document::Buck } } break; - case BucketDiskMoveCommand::ID: - // Fail bucket not found if op != MOVE - // Fail and log error if op == MOVE - { - api::BucketCommand& cmd(static_cast<api::BucketCommand&>(msg)); - if (cmd.getBucket() == source) { - if (op == MOVE) { - returnCode = api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE, - "Multiple bucket disk move commands pending at the same time " - " towards bucket " + source.toString()); - } else { - returnCode = api::ReturnCode(api::ReturnCode::BUCKET_DELETED, splitOrJoin(op)); - } - } - break; - } case ReadBucketInfo::ID: case RecheckBucketInfoCommand::ID: { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index cc4768cb91d..eec8ac3e327 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -704,15 +704,6 @@ FileStorManager::onInternal(const shared_ptr<api::InternalCommand>& msg) } return true; } - case BucketDiskMoveCommand::ID: - { - shared_ptr<BucketDiskMoveCommand> cmd(std::static_pointer_cast<BucketDiskMoveCommand>(msg)); - StorBucketDatabase::WrappedEntry entry(mapOperationToDisk(*cmd, cmd->getBucket())); - if (entry.exist()) { - handlePersistenceMessage(cmd, entry->disk); - } - return true; - } case RecheckBucketInfoCommand::ID: { shared_ptr<RecheckBucketInfoCommand> cmd(std::static_pointer_cast<RecheckBucketInfoCommand>(msg)); diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 4a99655c775..8897e3526ec 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -102,7 +102,6 @@ PersistenceThread::PersistenceThread(vespalib::ISequencedTaskExecutor * sequence _spi(provider), _processAllHandler(_env, provider), _mergeHandler(_spi, _env), - _diskMoveHandler(_env, _spi), _bucketOwnershipNotifier(), _flushMonitor(), _closed(false) @@ -810,8 +809,6 @@ PersistenceThread::handleCommandSplitByType(api::StorageCommand& msg, MessageTra return handleReadBucketList(static_cast<ReadBucketList&>(msg), std::move(tracker)); case ReadBucketInfo::ID: return handleReadBucketInfo(static_cast<ReadBucketInfo&>(msg), std::move(tracker)); - case BucketDiskMoveCommand::ID: - return _diskMoveHandler.handleBucketDiskMove(static_cast<BucketDiskMoveCommand&>(msg), std::move(tracker)); case InternalBucketJoinCommand::ID: return handleInternalBucketJoin(static_cast<InternalBucketJoinCommand&>(msg), std::move(tracker)); case RecheckBucketInfoCommand::ID: diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index 8eae0d8b34f..32387bc6826 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -5,9 +5,9 @@ #include "diskthread.h" #include "processallhandler.h" #include "mergehandler.h" -#include "diskmoveoperationhandler.h" #include "persistenceutil.h" #include "provider_error_wrapper.h" +#include <vespa/storage/common/bucketmessages.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/common/statusmessages.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> @@ -53,7 +53,6 @@ private: spi::PersistenceProvider& _spi; ProcessAllHandler _processAllHandler; MergeHandler _mergeHandler; - DiskMoveOperationHandler _diskMoveHandler; ServiceLayerComponent::UP _component; framework::Thread::UP _thread; std::unique_ptr<BucketOwnershipNotifier> _bucketOwnershipNotifier; diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.cpp b/storage/src/vespa/storage/storageserver/servicelayernode.cpp index 524e7c19d53..1b90e235d20 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernode.cpp @@ -13,7 +13,6 @@ #include <vespa/storage/visiting/visitormanager.h> #include <vespa/storage/bucketdb/bucketmanager.h> #include <vespa/storage/bucketdb/storagebucketdbinitializer.h> -#include <vespa/storage/bucketmover/bucketmover.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> #include <vespa/persistence/spi/exceptions.h> @@ -241,7 +240,6 @@ ServiceLayerNode::createChain() auto* merge_throttler = new MergeThrottler(_configUri, compReg); chain->push_back(StorageLink::UP(merge_throttler)); chain->push_back(StorageLink::UP(new ChangedBucketOwnershipHandler(_configUri, compReg))); - chain->push_back(StorageLink::UP(new bucketmover::BucketMover(_configUri, compReg))); chain->push_back(StorageLink::UP(new StorageBucketDBInitializer( _configUri, _partitions, getDoneInitializeHandler(), compReg))); chain->push_back(StorageLink::UP(new BucketManager(_configUri, _context.getComponentRegister()))); |