diff options
15 files changed, 56 insertions, 634 deletions
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 2a0ea6c7cb3..eaff8129234 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -134,7 +134,6 @@ vespa_define_module( src/tests/proton/reprocessing/document_reprocessing_handler src/tests/proton/reprocessing/reprocessing_runner src/tests/proton/server - src/tests/proton/server/data_directory_upgrader src/tests/proton/server/disk_mem_usage_filter src/tests/proton/server/health_adapter src/tests/proton/server/memory_flush_config_updater diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index 4bb1eb69996..98b4db85d36 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -1,15 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vespalib/util/signalhandler.h> -#include <vespa/vespalib/util/programoptions.h> -#include <string> -#include <iostream> + #include <vespa/searchcore/proton/server/proton.h> #include <vespa/searchlib/util/statefile.h> #include <vespa/searchlib/util/sigbushandler.h> #include <vespa/searchlib/util/ioerrorhandler.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/metrics/metricmanager.h> +#include <vespa/vespalib/util/signalhandler.h> +#include <vespa/vespalib/util/programoptions.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/fastos/app.h> + +#include <string> +#include <iostream> + #include <vespa/log/log.h> LOG_SETUP("proton"); @@ -43,8 +46,7 @@ App::setupSignals() SIG::TERM.hook(); } -namespace -{ +namespace { vespalib::string getStateString(search::StateFile &stateFile) @@ -61,7 +63,6 @@ bool stateIsDown(const vespalib::string &stateString) return strstr(stateString.c_str(), "state=down") != nullptr; } - } Params @@ -102,9 +103,9 @@ public: PersistenceProvider *downPersistence); ~ProtonServiceLayerProcess() { shutdown(); } - virtual void shutdown() override; - virtual void setupProvider() override; - virtual storage::spi::PersistenceProvider& getProvider() override; + void shutdown() override; + void setupProvider() override; + storage::spi::PersistenceProvider& getProvider() override; void setMetricManager(metrics::MetricManager& mm) { // The service layer will call init(...) and stop() on the metric @@ -114,7 +115,7 @@ public: // down component. _metricManager = &mm; } - virtual int64_t getGeneration() const override; + int64_t getGeneration() const override; }; ProtonServiceLayerProcess::ProtonServiceLayerProcess(const config::ConfigUri & @@ -179,8 +180,7 @@ App::Main() if (proton.hasAbortedInit()) { EV_STOPPING("proton", "shutdown after aborted init"); } else { - const ProtonConfig &protonConfig = - configSnapshot->getProtonConfig(); + const ProtonConfig &protonConfig = configSnapshot->getProtonConfig(); vespalib::string basedir = protonConfig.basedir; bool stopOnIOErrors = protonConfig.stoponioerrors; vespalib::mkdir(basedir, true); @@ -193,19 +193,15 @@ App::Main() LOG(error, "proton state string is %s", stateString.c_str()); if (stopOnIOErrors) { if ( !params.serviceidentity.empty()) { - downPersistence.reset( - new DownPersistence("proton state string is " + - stateString)); + downPersistence.reset(new DownPersistence("proton state string is " + stateString)); } else { LOG(info, "Sleeping 900 seconds due to proton state"); int sleepLeft = 900; - while (!(SIG::INT.check() || SIG::TERM.check()) && - sleepLeft > 0) { + while (!(SIG::INT.check() || SIG::TERM.check()) && sleepLeft > 0) { FastOS_Thread::Sleep(1000); --sleepLeft; } - EV_STOPPING("proton", - "shutdown after stop on io errors"); + EV_STOPPING("proton", "shutdown after stop on io errors"); return 1; } } @@ -236,20 +232,14 @@ App::Main() stateGen = stateFile->getGen(); stateString = getStateString(*stateFile); if (stateIsDown(stateString)) { - LOG(error, "proton state string is %s", - stateString.c_str()); + LOG(error, "proton state string is %s", stateString.c_str()); if (stopOnIOErrors) { if (spiProton) { // report down state to cluster controller. - spiProton->getNode(). - notifyPartitionDown(0, - "proton state " - "string is " + - stateString); + spiProton->getNode().notifyPartitionDown(0, "proton state string is " + stateString); FastOS_Thread::Sleep(1000); } - EV_STOPPING("proton", - "shutdown after new stop on io errors"); + EV_STOPPING("proton", "shutdown after new stop on io errors"); return 1; } } diff --git a/searchcore/src/testlist.txt b/searchcore/src/testlist.txt index bbe72775946..b74cd851052 100644 --- a/searchcore/src/testlist.txt +++ b/searchcore/src/testlist.txt @@ -59,7 +59,6 @@ tests/proton/reprocessing/attribute_reprocessing_initializer tests/proton/reprocessing/document_reprocessing_handler tests/proton/reprocessing/reprocessing_runner tests/proton/server -tests/proton/server/data_directory_upgrader tests/proton/server/disk_mem_usage_filter tests/proton/server/health_adapter tests/proton/server/memoryflush diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/.gitignore b/searchcore/src/tests/proton/server/data_directory_upgrader/.gitignore deleted file mode 100644 index b085eedc970..00000000000 --- a/searchcore/src/tests/proton/server/data_directory_upgrader/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_data_directory_upgrader_test_app diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt b/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt deleted file mode 100644 index ffcc7a82d2e..00000000000 --- a/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_data_directory_upgrader_test_app TEST - SOURCES - data_directory_upgrader_test.cpp - DEPENDS - searchcore_server - searchcore_flushengine -) -vespa_add_test(NAME searchcore_data_directory_upgrader_test_app COMMAND searchcore_data_directory_upgrader_test_app) diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/DESC b/searchcore/src/tests/proton/server/data_directory_upgrader/DESC deleted file mode 100644 index d0ca4c99210..00000000000 --- a/searchcore/src/tests/proton/server/data_directory_upgrader/DESC +++ /dev/null @@ -1 +0,0 @@ -data_directory_upgrader test. Take a look at data_directory_upgrader_test.cpp for details. diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/FILES b/searchcore/src/tests/proton/server/data_directory_upgrader/FILES deleted file mode 100644 index d1aee9bddfa..00000000000 --- a/searchcore/src/tests/proton/server/data_directory_upgrader/FILES +++ /dev/null @@ -1 +0,0 @@ -data_directory_upgrader_test.cpp diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/data_directory_upgrader_test.cpp b/searchcore/src/tests/proton/server/data_directory_upgrader/data_directory_upgrader_test.cpp deleted file mode 100644 index 7b6cf9143ee..00000000000 --- a/searchcore/src/tests/proton/server/data_directory_upgrader/data_directory_upgrader_test.cpp +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("data_directory_upgrader_test"); -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/searchcore/proton/server/data_directory_upgrader.h> -#include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <iostream> - -using namespace proton; -using namespace vespalib; - -typedef DataDirectoryUpgrader::RowColDir RowColDir; -typedef DataDirectoryUpgrader::ScanResult ScanResult; -typedef DataDirectoryUpgrader::UpgradeResult UpgradeResult; - -const string SCAN_DIR = "mytest"; -const string DEST_DIR = SCAN_DIR + "/n1"; - -void -assertDirs(const DirectoryList &exp, const DirectoryList &act) -{ - ASSERT_EQUAL(exp.size(), act.size()); - for (size_t i = 0; i < exp.size(); ++i) { - EXPECT_EQUAL(exp[i], act[i]); - } -} - -void -assertDirs(const DirectoryList &rowColDirs, bool destDirExisting, const ScanResult &act) -{ - ASSERT_EQUAL(rowColDirs.size(), act.getRowColDirs().size()); - for (size_t i = 0; i < rowColDirs.size(); ++i) { - EXPECT_EQUAL(rowColDirs[i], act.getRowColDirs()[i].dir()); - } - EXPECT_EQUAL(destDirExisting, act.isDestDirExisting()); -} - -void -assertDataFile(const vespalib::string &dir) -{ - FileInfo::UP file = stat(dir + "/data.txt"); - ASSERT_TRUE(file.get() != NULL); - EXPECT_TRUE(file->_plainfile); -} - -vespalib::string -readFile(const vespalib::string &fileName) -{ - File file(fileName); - file.open(File::READONLY); - FileInfo info = file.stat(); - char buf[512]; - size_t bytesRead = file.read(&buf, info._size, 0); - return vespalib::string(buf, bytesRead); -} - -void -assertUpgradeFile(const vespalib::string &exp, const vespalib::string &dir) -{ - EXPECT_EQUAL(exp, readFile(dir + "/data-directory-upgrade-source.txt")); -} - -void -assertDowngradeScript(const vespalib::string &exp, const vespalib::string &dir) -{ - EXPECT_EQUAL(exp, readFile(dir + "/data-directory-downgrade.sh")); -} - -struct BaseFixture -{ - DataDirectoryUpgrader _upg; - BaseFixture(const DirectoryList &dirs, bool createDestDir = false) : _upg(SCAN_DIR, DEST_DIR) { - mkdir(SCAN_DIR); - if (createDestDir) { - mkdir(DEST_DIR); - } - for (const string &dir : dirs) { - mkdir(SCAN_DIR + "/" + dir); - File f(SCAN_DIR + "/" + dir + "/data.txt"); - f.open(File::CREATE); - f.close(); - } - } - virtual ~BaseFixture() { - rmdir(SCAN_DIR, true); - } - DirectoryList getDirs(const vespalib::string &subDir = "") const { - DirectoryList l = listDirectory(SCAN_DIR + "/" + subDir); - std::sort(l.begin(), l.end()); - return l; - } -}; - -struct EmptyFixture : public BaseFixture -{ - EmptyFixture() : BaseFixture({}) {} -}; - -struct SingleFixture : public BaseFixture -{ - SingleFixture() : BaseFixture({"r0/c0"}) {} -}; - -struct DoubleFixture : public BaseFixture -{ - DoubleFixture() : BaseFixture({"r0/c0", "r1/c1"}) {} -}; - -struct UnrelatedFixture : public BaseFixture -{ - UnrelatedFixture() : BaseFixture({"r0/cY", "rX/c1", "r0"}) {} -}; - -struct ExistingDestinationFixture : public BaseFixture -{ - ExistingDestinationFixture() : BaseFixture({"r0/c0"}, true) {} -}; - -TEST_F("require that single row/column directory is discovered", SingleFixture) -{ - ScanResult res = f._upg.scan(); - assertDirs({"r0/c0"}, false, res); -} - -TEST_F("require that multiple row/column directories are discovered", DoubleFixture) -{ - ScanResult res = f._upg.scan(); - assertDirs({"r0/c0", "r1/c1"}, false, res); -} - -TEST_F("require that unrelated directories are not discovered", UnrelatedFixture) -{ - ScanResult res = f._upg.scan(); - assertDirs({}, false, res); -} - -TEST_F("require that existing destination directory is discovered", ExistingDestinationFixture) -{ - ScanResult res = f._upg.scan(); - assertDirs({"r0/c0"}, true, res); -} - -TEST("require that no-existing scan directory is handled") -{ - DataDirectoryUpgrader upg(SCAN_DIR, DEST_DIR); - ScanResult res = upg.scan(); - assertDirs({}, false, res); -} - -TEST_F("require that empty directory is left untouched", EmptyFixture) -{ - UpgradeResult res = f._upg.upgrade(f._upg.scan()); - EXPECT_EQUAL(DataDirectoryUpgrader::IGNORE, res.getStatus()); - EXPECT_EQUAL("No directory to upgrade", res.getDesc()); - DirectoryList dirs = f.getDirs(); - assertDirs({}, dirs); -} - -TEST_F("require that existing destination directory is left untouched", ExistingDestinationFixture) -{ - UpgradeResult res = f._upg.upgrade(f._upg.scan()); - EXPECT_EQUAL(DataDirectoryUpgrader::IGNORE, res.getStatus()); - EXPECT_EQUAL("Destination directory 'mytest/n1' is already existing", res.getDesc()); - DirectoryList dirs = f.getDirs(); - assertDirs({"n1", "r0"}, dirs); -} - -TEST_F("require that single directory is upgraded", SingleFixture) -{ - UpgradeResult res = f._upg.upgrade(f._upg.scan()); - EXPECT_EQUAL(DataDirectoryUpgrader::COMPLETE, res.getStatus()); - EXPECT_EQUAL("Moved data from 'mytest/r0/c0' to 'mytest/n1'", res.getDesc()); - DirectoryList dirs = f.getDirs(); - std::sort(dirs.begin(), dirs.end()); - assertDirs({"n1"}, dirs); - assertDataFile(DEST_DIR); - assertUpgradeFile("mytest/r0/c0", DEST_DIR); - assertDowngradeScript("#!/bin/sh\n\n" - "mkdir mytest/r0 || exit 1\n" - "chown yahoo mytest/r0\n" - "mv mytest/n1 mytest/r0/c0\n" - "rm mytest/r0/c0/data-directory-upgrade-source.txt\n" - "rm mytest/r0/c0/data-directory-downgrade.sh\n", DEST_DIR); -} - -TEST_F("require that multiple directories are left untouched", DoubleFixture) -{ - UpgradeResult res = f._upg.upgrade(f._upg.scan()); - EXPECT_EQUAL(DataDirectoryUpgrader::ERROR, res.getStatus()); - EXPECT_EQUAL("Can only upgrade a single directory, was asked to upgrade 2 ('r0/c0', 'r1/c1')", res.getDesc()); - DirectoryList dirs = f.getDirs(); - std::sort(dirs.begin(), dirs.end()); - assertDirs({"r0", "r1"}, dirs); - assertDataFile(SCAN_DIR + "/r0/c0"); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 18c000e5f4e..8333e180a0a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -7,7 +7,6 @@ vespa_add_library(searchcore_server STATIC bucketmovejob.cpp clusterstatehandler.cpp combiningfeedview.cpp - data_directory_upgrader.cpp ddbstate.cpp disk_mem_usage_filter.cpp disk_mem_usage_sampler.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.cpp b/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.cpp deleted file mode 100644 index 3238f67353c..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.cpp +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "data_directory_upgrader.h" - -#include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/exceptions.h> -#include <iostream> -#include <sys/stat.h> - -namespace proton { - -namespace { - -vespalib::string UPGRADE_SOURCE_FILE = "data-directory-upgrade-source.txt"; -vespalib::string DOWNGRADE_SCRIPT_FILE = "data-directory-downgrade.sh"; - -bool -isValidDir(const vespalib::string &dir, char prefix) -{ - if (dir.empty() || dir[0] != prefix) { - return false; - } - vespalib::asciistream stream(dir.substr(1)); - uint32_t number = 0; - try { - stream >> number; - } catch (const vespalib::IllegalArgumentException &) { - return false; - } - return true; -} - -bool -isRowDir(const vespalib::string &dir) -{ - return isValidDir(dir, 'r'); -} - -bool -isColumnDir(const vespalib::string &dir) -{ - return isValidDir(dir, 'c'); -} - -vespalib::string -createDirString(const DataDirectoryUpgrader::RowColDirs &dirs) -{ - vespalib::asciistream result; - bool first = true; - for (const auto &dir : dirs) { - if (!first) { - result << ", "; - } - result << "'" << dir.dir() << "'"; - first = false; - } - return result.str(); -} - -void -writeUpgradeFile(const vespalib::string &srcDir, - const vespalib::string &dstDir) -{ - vespalib::File file(dstDir + "/" + UPGRADE_SOURCE_FILE); - file.open(vespalib::File::CREATE); - file.write(&srcDir[0], srcDir.size(), 0); - file.close(); -} - -void -writeDowngradeScript(const vespalib::string &scanDir, - const vespalib::string &dstDir, - const DataDirectoryUpgrader::RowColDir &rowColDir) -{ - vespalib::asciistream script; - vespalib::string fullRowDir = scanDir + "/" + rowColDir.row(); - vespalib::string fullRowColDir = scanDir + "/" + rowColDir.dir(); - script << "#!/bin/sh\n\n"; - script << "mkdir " << fullRowDir << " || exit 1\n"; - script << "chown yahoo " << fullRowDir << "\n"; - script << "mv " << dstDir << " " << fullRowColDir << "\n"; - script << "rm " << fullRowColDir << "/" << UPGRADE_SOURCE_FILE << "\n"; - script << "rm " << fullRowColDir << "/" << DOWNGRADE_SCRIPT_FILE << "\n"; - vespalib::string fileName = dstDir + "/" + DOWNGRADE_SCRIPT_FILE; - vespalib::File file(fileName); - file.open(vespalib::File::CREATE); - file.write(script.c_str(), script.size(), 0); - file.close(); - chmod(fileName.c_str(), 0755); -} - -} - - -DataDirectoryUpgrader::RowColDir::RowColDir(const vespalib::string &row_, - const vespalib::string &col_) - : _row(row_), - _col(col_) -{ -} - -DataDirectoryUpgrader::RowColDir::~RowColDir() { } - -DataDirectoryUpgrader::ScanResult::ScanResult() - : _rowColDirs(), - _destDirExisting(false) -{ -} - -DataDirectoryUpgrader::UpgradeResult::UpgradeResult(const Status status, - const vespalib::string &desc) - : _status(status), - _desc(desc) -{ -} - -DataDirectoryUpgrader::DataDirectoryUpgrader(const vespalib::string &scanDir, - const vespalib::string &destDir) - : _scanDir(scanDir), - _destDir(destDir) -{ -} - -DataDirectoryUpgrader::~DataDirectoryUpgrader() {} - -DataDirectoryUpgrader::ScanResult -DataDirectoryUpgrader::scan() const -{ - ScanResult result; - try { - vespalib::DirectoryList dirs = listDirectory(_scanDir); - for (const auto &dir : dirs) { - if (isRowDir(dir)) { - vespalib::DirectoryList subDirs = listDirectory(_scanDir + "/" + dir); - for (const auto &subDir : subDirs) { - if (isColumnDir(subDir)) { - result.addDir(RowColDir(dir, subDir)); - } - } - } - } - } catch (const vespalib::IoException &) { - // Scan dir does not exists - } - try { - if (vespalib::stat(_destDir).get() != NULL) { - result.setDestDirExisting(true); - } - } catch (const vespalib::IoException &) {} - std::sort(result.getRowColDirs().begin(), result.getRowColDirs().end()); - return result; -} - -DataDirectoryUpgrader::UpgradeResult -DataDirectoryUpgrader::upgrade(const ScanResult &scanResult) const -{ - if (scanResult.isDestDirExisting()) { - return UpgradeResult(IGNORE, - vespalib::make_string("Destination directory '%s' is already existing", - _destDir.c_str())); - } - const RowColDirs &rowColDirs = scanResult.getRowColDirs(); - if (rowColDirs.empty()) { - return UpgradeResult(IGNORE, "No directory to upgrade"); - } - if (rowColDirs.size() > 1) { - return UpgradeResult(ERROR, - vespalib::make_string("Can only upgrade a single directory, was asked to upgrade %zu (%s)", - rowColDirs.size(), createDirString(rowColDirs).c_str())); - } - const vespalib::string src = _scanDir + "/" + rowColDirs[0].dir(); - const vespalib::string &dst = _destDir; - try { - if (!vespalib::rename(src, dst)) { - return UpgradeResult(ERROR, - vespalib::make_string("Failed to rename directory '%s' to '%s'", - src.c_str(), dst.c_str())); - } - const vespalib::string rmDir = _scanDir + "/" + rowColDirs[0].row(); - if (!vespalib::rmdir(rmDir, false)) { - return UpgradeResult(ERROR, - vespalib::make_string("Failed to remove empty directory '%s'", - rmDir.c_str())); - } - writeUpgradeFile(src, dst); - writeDowngradeScript(_scanDir, dst, rowColDirs[0]); - } catch (const vespalib::IoException &ex) { - return UpgradeResult(ERROR, - vespalib::make_string("Got exception during data directory upgrade from '%s' to '%s': %s", - src.c_str(), dst.c_str(), ex.what())); - } - return UpgradeResult(COMPLETE, - vespalib::make_string("Moved data from '%s' to '%s'", - src.c_str(), dst.c_str())); -} - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.h b/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.h deleted file mode 100644 index 66963dd8f19..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.h +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <vector> - -namespace proton { - -/** - * Class used to upgrade a row column directory /rX/cY to an elastic directory /nZ - * where Z is the distribution key for that search node. - */ -class DataDirectoryUpgrader -{ -public: - class RowColDir - { - private: - vespalib::string _row; - vespalib::string _col; - - public: - RowColDir(const vespalib::string &row_, const vespalib::string &col_); - ~RowColDir(); - const vespalib::string &row() const { return _row; } - const vespalib::string &col() const { return _col; } - vespalib::string dir() const { return row() + "/" + col(); } - bool operator< (const RowColDir &rhs) const { return dir() < rhs.dir(); } - }; - typedef std::vector<RowColDir> RowColDirs; - - class ScanResult - { - private: - RowColDirs _rowColDirs; - bool _destDirExisting; - - public: - ScanResult(); - void addDir(const RowColDir &dir) { - _rowColDirs.push_back(dir); - } - RowColDirs &getRowColDirs() { return _rowColDirs; } - const RowColDirs &getRowColDirs() const { return _rowColDirs; } - void setDestDirExisting(bool val) { _destDirExisting = val; } - bool isDestDirExisting() const { return _destDirExisting; } - }; - - enum Status - { - IGNORE, - COMPLETE, - ERROR - }; - - class UpgradeResult - { - private: - const Status _status; - const vespalib::string _desc; - - public: - UpgradeResult(const Status status, const vespalib::string &desc); - Status getStatus() const { return _status; } - const vespalib::string &getDesc() const { return _desc; } - }; - -private: - const vespalib::string _scanDir; - const vespalib::string _destDir; - -public: - DataDirectoryUpgrader(const vespalib::string &scanDir, const vespalib::string &destDir); - ~DataDirectoryUpgrader(); - ScanResult scan() const; - UpgradeResult upgrade(const ScanResult &scanResult) const; -}; - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index f4a682af170..a8b4256ea16 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -651,9 +651,8 @@ DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot) _config_store->saveConfig(configSnapshot, confSerial); } - void -DocumentDB::resumeSaveConfig(void) +DocumentDB::resumeSaveConfig() { SerialNum bestSerial = _config_store->getBestSerialNum(); if (bestSerial == 0) @@ -692,14 +691,12 @@ DocumentDB::onPerformPrune(SerialNum flushedSerial) _config_store->prune(flushedSerial); } - bool -DocumentDB::getAllowPrune(void) const +DocumentDB::getAllowPrune() const { return _state.getAllowPrune(); } - void DocumentDB::start() { @@ -710,14 +707,12 @@ DocumentDB::start() internalInit(); } - void DocumentDB::waitForInitDone() { _initGate.await(); } - void DocumentDB::startTransactionLogReplay() { @@ -737,9 +732,7 @@ DocumentDB::startTransactionLogReplay() *_config_store); _initGate.countDown(); - LOG(debug, - "DocumentDB(%s): Database started.", - _docTypeName.toString().c_str()); + LOG(debug, "DocumentDB(%s): Database started.", _docTypeName.toString().c_str()); } BucketGuard::UP DocumentDB::lockBucket(const document::BucketId &bucket) @@ -814,15 +807,13 @@ DocumentDB::setIndexSchema(const DocumentDBConfig &configSnapshot, SerialNum ser // TODO: Adjust tune. } - void DocumentDB::reconfigure(const DocumentDBConfig::SP & snapshot) { masterExecute([this, snapshot]() { newConfigSnapshot(snapshot); }); // Wait for config to be applied, or for document db close std::unique_lock<std::mutex> guard(_configMutex); - while ((_activeConfigSnapshotGeneration < snapshot->getGeneration()) && - !_state.getClosed()) { + while ((_activeConfigSnapshotGeneration < snapshot->getGeneration()) && !_state.getClosed()) { _configCV.wait(guard); } } @@ -880,21 +871,17 @@ DocumentDB::reportStatus() const float progress = _feedHandler.getReplayProgress() * 100.0f; vespalib::string msg = vespalib::make_string("DocumentDB replay transaction log on startup (%u%% done)", static_cast<uint32_t>(progress)); - return StatusReport::create(params.state(StatusReport::PARTIAL). - progress(progress). - message(msg)); + return StatusReport::create(params.state(StatusReport::PARTIAL).progress(progress).message(msg)); } else if (rawState == DDBState::State::APPLY_LIVE_CONFIG) { - return StatusReport::create(params.state(StatusReport::PARTIAL). - message("DocumentDB apply live config on startup")); + return StatusReport::create(params.state(StatusReport::PARTIAL) + .message("DocumentDB apply live config on startup")); } else if (rawState == DDBState::State::REPROCESS || rawState == DDBState::State::REDO_REPROCESS) { float progress = _subDBs.getReprocessingProgress() * 100.0f; vespalib::string msg = make_string("DocumentDB reprocess on startup (%u%% done)", - static_cast<uint32_t>(progress)); - return StatusReport::create(params.state(StatusReport::PARTIAL). - progress(progress). - message(msg)); + static_cast<uint32_t>(progress)); + return StatusReport::create(params.state(StatusReport::PARTIAL).progress(progress).message(msg)); } else if (_state.getDelayedConfig()) { return StatusReport::create(params.state(StatusReport::PARTIAL). message("DocumentDB delaying attribute aspects changes in config")); @@ -947,15 +934,13 @@ DocumentDB::listSchema(std::vector<vespalib::string> &fieldNames, fieldLocations); } - int64_t DocumentDB::getActiveGeneration() const { lock_guard guard(_configMutex); return _activeConfigSnapshotGeneration; } - void -DocumentDB::syncFeedView(void) +DocumentDB::syncFeedView() { // Called by executor or while in rendezvous with executor @@ -973,14 +958,12 @@ DocumentDB::syncFeedView(void) performDropFeedView(oldFeedView); } - bool DocumentDB::hasDocument(const document::DocumentId &id) { return _subDBs.getReadySubDB()->hasDocument(id); } - void DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) { @@ -1009,7 +992,7 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) } void -DocumentDB::performStartMaintenance(void) +DocumentDB::performStartMaintenance() { // Called by executor thread // Only start once, after replay done @@ -1030,13 +1013,13 @@ DocumentDB::performStartMaintenance(void) } void -DocumentDB::stopMaintenance(void) +DocumentDB::stopMaintenance() { _maintenanceController.stop(); } void -DocumentDB::forwardMaintenanceConfig(void) +DocumentDB::forwardMaintenanceConfig() { // Called by executor thread DocumentDBConfig::SP activeConfig = getActiveConfig(); @@ -1053,8 +1036,7 @@ DocumentDB::forwardMaintenanceConfig(void) } void -DocumentDB::notifyClusterStateChanged( - const IBucketStateCalculator::SP &newCalc) +DocumentDB::notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) { // Called by executor thread _calc = newCalc; // Save for maintenance job injection @@ -1062,8 +1044,7 @@ DocumentDB::notifyClusterStateChanged( IFeedView::SP feedView(_feedView.get()); if (feedView.get() != NULL) { // Try downcast to avoid polluting API - CombiningFeedView *cfv = dynamic_cast<CombiningFeedView *> - (feedView.get()); + CombiningFeedView *cfv = dynamic_cast<CombiningFeedView *>(feedView.get()); if (cfv != NULL) cfv->setCalculator(newCalc); } @@ -1376,12 +1357,9 @@ DocumentDB::updateMetrics(DocumentDBTaggedMetrics &metrics) _jobTrackers.updateMetrics(metrics.job); updateMetrics(metrics.attribute); - updateDocumentStoreMetrics(metrics.ready.documentStore, - _subDBs.getReadySubDB()); - updateDocumentStoreMetrics(metrics.removed.documentStore, - _subDBs.getRemSubDB()); - updateDocumentStoreMetrics(metrics.notReady.documentStore, - _subDBs.getNotReadySubDB()); + updateDocumentStoreMetrics(metrics.ready.documentStore, _subDBs.getReadySubDB()); + updateDocumentStoreMetrics(metrics.removed.documentStore, _subDBs.getRemSubDB()); + updateDocumentStoreMetrics(metrics.notReady.documentStore, _subDBs.getNotReadySubDB()); DocumentMetaStoreReadGuards dmss(_subDBs); updateLidSpaceMetrics(metrics.ready.lidSpace, dmss.readydms->get()); updateLidSpaceMetrics(metrics.notReady.lidSpace, dmss.notreadydms->get()); @@ -1391,13 +1369,10 @@ DocumentDB::updateMetrics(DocumentDBTaggedMetrics &metrics) void DocumentDB::sync(SerialNum syncTo) { - LOG(spam, - "DocumentDB(%s): sync(): serialNum=%" PRIu64, - _docTypeName.toString().c_str(), syncTo); + LOG(spam, "DocumentDB(%s): sync(): serialNum=%zu", _docTypeName.toString().c_str(), syncTo); _feedHandler.syncTls(syncTo); } - void DocumentDB::waitForOnlineState() { @@ -1410,5 +1385,4 @@ DocumentDB::getDistributionKey() const return _owner.getDistributionKey(); } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 8c5e64500ce..93d26e3406d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -154,7 +154,7 @@ private: /** * Resume interrupted config save if needed. */ - void resumeSaveConfig(void); + void resumeSaveConfig(); void setIndexSchema(const DocumentDBConfig &configSnapshot, SerialNum serialNum); @@ -186,7 +186,7 @@ private: /** * Implements IFeedHandlerOwner **/ - virtual bool getAllowPrune(void) const override; + virtual bool getAllowPrune() const override; void startTransactionLogReplay(); @@ -292,7 +292,7 @@ public: * * @return metrics update hook **/ - metrics::UpdateHook & getMetricsUpdateHook(void) { + metrics::UpdateHook & getMetricsUpdateHook() { return _metricsHook; } @@ -328,12 +328,12 @@ public: /** * Returns the feed handler for this database. */ - FeedHandler & getFeedHandler(void) { return _feedHandler; } + FeedHandler & getFeedHandler() { return _feedHandler; } /** * Returns the bucket handler for this database. */ - BucketHandler & getBucketHandler(void) { return _bucketHandler; } + BucketHandler & getBucketHandler() { return _bucketHandler; } /** * Returns the cluster state handler for this database. @@ -395,7 +395,7 @@ public: */ virtual void replayConfig(SerialNum serialNum) override; - const DocTypeName & getDocTypeName(void) const { return _docTypeName; } + const DocTypeName & getDocTypeName() const { return _docTypeName; } void listSchema(std::vector<vespalib::string> &fieldNames, @@ -423,9 +423,9 @@ public: * Implements IFeedHandlerOwner **/ void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config); - void performStartMaintenance(void); - void stopMaintenance(void); - void forwardMaintenanceConfig(void); + void performStartMaintenance(); + void stopMaintenance(); + void forwardMaintenanceConfig(); /** * Updates metrics collection object, and resets executor stats. diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index ecfb54682ff..9e16cc63def 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "data_directory_upgrader.h" #include "disk_mem_usage_sampler.h" #include "document_db_explorer.h" #include "flushhandlerproxy.h" @@ -221,11 +220,6 @@ Proton::init() auto bootstrapConfig = configSnapshot->getBootstrapConfig(); assert(bootstrapConfig); - const ProtonConfig &protonConfig = bootstrapConfig->getProtonConfig(); - - if (!performDataDirectoryUpgrade(protonConfig.basedir)) { - _abortInit = true; - } return bootstrapConfig; } @@ -340,40 +334,6 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _initComplete = true; } -bool -Proton::performDataDirectoryUpgrade(const vespalib::string &baseDir) -{ - // TODO: Remove this functionality when going to Vespa 6. - vespalib::string scanDir = baseDir.substr(0, baseDir.rfind('/')); - LOG(debug, "About to perform data directory upgrade: scanDir='%s', destDir='%s'", - scanDir.c_str(), baseDir.c_str()); - DataDirectoryUpgrader upgrader(scanDir, baseDir); - DataDirectoryUpgrader::ScanResult scanResult = upgrader.scan(); - DataDirectoryUpgrader::UpgradeResult upgradeResult = upgrader.upgrade(scanResult); - if (upgradeResult.getStatus() == DataDirectoryUpgrader::ERROR) { - LOG(error, "Data directory upgrade failed: '%s'. Please consult Vespa release notes on how to manually fix this issue. " - "The search node will not start until this issue has been fixed", upgradeResult.getDesc().c_str()); - return false; - } else if (upgradeResult.getStatus() == DataDirectoryUpgrader::IGNORE) { - LOG(debug, "Data directory upgrade ignored: %s", upgradeResult.getDesc().c_str()); - } else if (upgradeResult.getStatus() == DataDirectoryUpgrader::COMPLETE) { - LOG(info, "Data directory upgrade completed: %s", upgradeResult.getDesc().c_str()); - } - return true; -} - -void -Proton::loadLibrary(const vespalib::string &libName) -{ - searchcorespi::IIndexManagerFactory::SP factory(_libraries.create(libName)); - if (factory.get() != NULL) { - LOG(info, "Successfully created index manager factory from library '%s'", libName.c_str()); - _indexManagerFactoryRegistry.add(libName, factory); - } else { - LOG(error, "Failed creating index manager factory from library '%s'", libName.c_str()); - } -} - searchcorespi::IIndexManagerFactory::SP Proton::getIndexManagerFactory(const vespalib::stringref & name) const { @@ -539,7 +499,7 @@ size_t Proton::getNumActiveDocs() const vespalib::string -Proton::getDelayedConfigs(void) const +Proton::getDelayedConfigs() const { std::ostringstream res; bool first = true; @@ -891,7 +851,7 @@ Proton::getComponentConfig(Consumer &consumer) } int64_t -Proton::getConfigGeneration(void) +Proton::getConfigGeneration() { return _protonConfigurer.getActiveConfigSnapshot()->getBootstrapConfig()->getGeneration(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 9c107d92c3c..25c490418a2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -145,9 +145,6 @@ private: std::unique_ptr<HwInfoSampler> _hwInfoSampler; std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry; - bool performDataDirectoryUpgrade(const vespalib::string &baseDir); - void loadLibrary(const vespalib::string &libName); - virtual IDocumentDBConfigOwner *addDocumentDB(const DocTypeName & docTypeName, const vespalib::string & configid, const BootstrapConfig::SP & bootstrapConfig, @@ -192,18 +189,15 @@ public: * * 1st phase init: start cheap clock thread and get initial config */ - BootstrapConfig::SP - init(); + BootstrapConfig::SP init(); /* * 2nd phase init: setup data structures. */ - void - init(const BootstrapConfig::SP & configSnapshot); + void init(const BootstrapConfig::SP & configSnapshot); - DocumentDB::SP - getDocumentDB(const document::DocumentType &docType); + DocumentDB::SP getDocumentDB(const document::DocumentType &docType); DocumentDB::SP addDocumentDB(const document::DocumentType &docType, @@ -235,18 +229,17 @@ public: /** * Return the oldest active config generation used by proton. */ - int64_t getConfigGeneration(void); + int64_t getConfigGeneration(); size_t getNumDocs() const; size_t getNumActiveDocs() const; DocsumBySlime & getDocsumBySlime() { return *_docsumBySlime; } - vespalib::string getDelayedConfigs(void) const; + vespalib::string getDelayedConfigs() const; virtual StatusReport::List getStatusReports() const override; MatchEngine & getMatchEngine() { return *_matchEngine; } - FlushEngine & getFlushEngine() { return *_flushEngine; } vespalib::ThreadStackExecutorBase & getExecutor() { return _executor; } bool isReplayDone() const { return _isReplayDone; } @@ -265,4 +258,3 @@ public: }; } // namespace proton - |