diff options
author | Tor Egge <Tor.Egge@broadpark.no> | 2018-07-31 14:34:29 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-07-31 14:34:29 +0200 |
commit | ccf197e50ef375abf16a879f2c66ea020609bcf4 (patch) | |
tree | 2a8e47f3688d0ed3c01f87608636f89266834995 | |
parent | ecae72d020d104155e4e68bd5adbf6d5b5ada3c1 (diff) | |
parent | bad7c179ba16030e94591a75155173edb483b619 (diff) |
Merge pull request #6490 from vespa-engine/toregge/remove-unused-document-db-dirs
Remove unused document db dirs
5 files changed, 278 insertions, 3 deletions
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 89e6493cfbc..9d996d96dc7 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -129,6 +129,7 @@ vespa_define_module( src/tests/proton/proton src/tests/proton/proton_config_fetcher src/tests/proton/proton_configurer + src/tests/proton/proton_disk_layout src/tests/proton/reference/gid_to_lid_change_handler src/tests/proton/reference/gid_to_lid_change_listener src/tests/proton/reference/gid_to_lid_change_registrator diff --git a/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt b/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt new file mode 100644 index 00000000000..f63fa21a954 --- /dev/null +++ b/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_proton_disk_layout_test_app TEST + SOURCES + proton_disk_layout_test.cpp + DEPENDS + searchcore_server + searchcore_fconfig +) +vespa_add_test(NAME searchcore_proton_disk_layout_test_app COMMAND searchcore_proton_disk_layout_test_app) diff --git a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp new file mode 100644 index 00000000000..edb4250ce76 --- /dev/null +++ b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp @@ -0,0 +1,178 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/server/proton_disk_layout.h> +#include <vespa/searchcore/proton/common/doctypename.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/transactionlog/translogserver.h> +#include <vespa/searchlib/transactionlog/translogclient.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/util/stringfmt.h> + +using search::index::DummyFileHeaderContext; +using search::transactionlog::TransLogClient; +using search::transactionlog::TransLogServer; +using proton::DocTypeName; +using proton::ProtonDiskLayout; + +static constexpr unsigned int tlsPort = 9018; + +static const vespalib::string baseDir("testdb"); +static const vespalib::string documentsDir(baseDir + "/documents"); + +struct FixtureBase +{ + FixtureBase() { vespalib::rmdir(baseDir, true); } + ~FixtureBase() { vespalib::rmdir(baseDir, true); } +}; + +struct DiskLayoutFixture { + DummyFileHeaderContext _fileHeaderContext; + TransLogServer _tls; + vespalib::string _tlsSpec; + ProtonDiskLayout _diskLayout; + + DiskLayoutFixture(); + ~DiskLayoutFixture(); + + void createDirs(const std::set<vespalib::string> &dirs) { + for (const auto &dir : dirs) { + vespalib::mkdir(documentsDir + "/" + dir, false); + } + } + void createDomains(const std::set<vespalib::string> &domains) { + TransLogClient tlc(_tlsSpec); + for (const auto &domain : domains) { + ASSERT_TRUE(tlc.create(domain)); + } + } + + std::set<vespalib::string> listDomains() { + std::vector<vespalib::string> domainVector; + TransLogClient tlc(_tlsSpec); + ASSERT_TRUE(tlc.listDomains(domainVector)); + std::set<vespalib::string> domains; + for (const auto &domain : domainVector) { + domains.emplace(domain); + } + return domains; + } + + std::set<vespalib::string> listDirs() { + std::set<vespalib::string> dirs; + auto names = vespalib::listDirectory(documentsDir); + for (const auto &name : names) { + if (vespalib::isDirectory(documentsDir + "/" + name)) { + dirs.emplace(name); + } + } + return dirs; + } + + void initAndPruneUnused(const std::set<vespalib::string> names) + { + std::set<DocTypeName> docTypeNames; + for (const auto &name: names) { + docTypeNames.emplace(name); + } + _diskLayout.initAndPruneUnused(docTypeNames); + } + + void assertDirs(const std::set<vespalib::string> &expDirs) { + EXPECT_EQUAL(expDirs, listDirs()); + } + + void assertDomains(const std::set<vespalib::string> &expDomains) + { + EXPECT_EQUAL(expDomains, listDomains()); + } +}; + +DiskLayoutFixture::DiskLayoutFixture() + : _fileHeaderContext(), + _tls("tls", tlsPort, baseDir, _fileHeaderContext), + _tlsSpec(vespalib::make_string("tcp/localhost:%u", tlsPort)), + _diskLayout(baseDir, _tlsSpec) +{ +} + +DiskLayoutFixture::~DiskLayoutFixture() = default; + +struct Fixture : public FixtureBase, public DiskLayoutFixture +{ + Fixture() + : FixtureBase(), + DiskLayoutFixture() + { + } +}; + +TEST_F("require that empty config is ok", Fixture) { + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that disk layout is preserved", FixtureBase) +{ + { + DiskLayoutFixture diskLayout; + diskLayout.createDirs({"foo", "bar"}); + diskLayout.createDomains({"bar", "baz"}); + } + { + DiskLayoutFixture diskLayout; + TEST_DO(diskLayout.assertDirs({"foo", "bar"})); + TEST_DO(diskLayout.assertDomains({"bar", "baz"})); + } +} + +TEST_F("require that used dir is preserved", Fixture) +{ + f.createDirs({"foo"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({"foo"})); + TEST_DO(f.assertDomains({"foo"})); +} + +TEST_F("require that unused dir is removed", Fixture) +{ + f.createDirs({"foo"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"bar"}); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that interrupted remove is completed", Fixture) +{ + f.createDirs({"foo.removed"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that early interrupted remove is completed", Fixture) +{ + f.createDirs({"foo", "foo.removed"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that live document db dir remove works", Fixture) +{ + f.createDirs({"foo"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({"foo"})); + TEST_DO(f.assertDomains({"foo"})); + f._diskLayout.remove(DocTypeName("foo")); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp index 00a7e9b9140..c2624719d81 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp @@ -44,6 +44,7 @@ TransLogServerMetrics::considerRemoveDomains(const DomainStats &stats) for (auto itr = _domainMetrics.begin(); itr != _domainMetrics.end(); ) { const vespalib::string &documentType = itr->first; if (stats.find(documentType) == stats.end()) { + _parent->unregisterMetric(*itr->second); itr = _domainMetrics.erase(itr); } else { ++itr; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp index 1bc24d4a0e3..ab4a0576ab0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp @@ -3,15 +3,75 @@ #include "proton_disk_layout.h" #include <vespa/vespalib/io/fileutil.h> #include <vespa/fastos/file.h> +#include <vespa/searchcore/proton/common/doctypename.h> +#include <vespa/searchlib/transactionlog/translogclient.h> #include <cassert> +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.proton_disk_layout"); + +using search::transactionlog::TransLogClient; + namespace proton { +namespace { + +struct DocumentDBDirMeta +{ + bool normal; + bool removed; + + DocumentDBDirMeta() + : normal(false), + removed(false) + { + } +}; + +using DocumentDBDirScan = std::map<DocTypeName, DocumentDBDirMeta>; + +vespalib::string getDocumentsDir(const vespalib::string &baseDir) +{ + return baseDir + "/documents"; +} + +vespalib::string removedSuffix(".removed"); + +vespalib::string getNormalName(const vespalib::string removedName) { + return removedName.substr(0, removedName.size() - removedSuffix.size()); +} + +vespalib::string getRemovedName(const vespalib::string &normalName) +{ + return normalName + removedSuffix; +} + +bool isRemovedName(const vespalib::string &dirName) +{ + return dirName.size() > removedSuffix.size() && dirName.substr(dirName.size() - removedSuffix.size()) == removedSuffix; +} + +void scanDir(const vespalib::string documentsDir, DocumentDBDirScan &dirs) +{ + auto names = vespalib::listDirectory(documentsDir); + for (const auto &name : names) { + if (vespalib::isDirectory(documentsDir + "/" + name)) { + if (isRemovedName(name)) { + dirs[DocTypeName(getNormalName(name))].removed = true; + } else { + dirs[DocTypeName(name)].normal = true; + } + } + } +} + +} + ProtonDiskLayout::ProtonDiskLayout(const vespalib::string &baseDir, const vespalib::string &tlsSpec) : _baseDir(baseDir), _tlsSpec(tlsSpec) { - vespalib::mkdir(_baseDir + "/documents", true); + vespalib::mkdir(getDocumentsDir(_baseDir), true); } ProtonDiskLayout::~ProtonDiskLayout() = default; @@ -19,13 +79,39 @@ ProtonDiskLayout::~ProtonDiskLayout() = default; void ProtonDiskLayout::remove(const DocTypeName &docTypeName) { - (void) docTypeName; + vespalib::string documentsDir(getDocumentsDir(_baseDir)); + vespalib::string name(docTypeName.toString()); + vespalib::string normalDir(documentsDir + "/" + name); + vespalib::string removedDir(documentsDir + "/" + getRemovedName(name)); + vespalib::rename(normalDir, removedDir, false, false); + TransLogClient tlc(_tlsSpec); + if (!tlc.remove(name)) { + LOG(fatal, "Failed to remove tls domain %s", name.c_str()); + LOG_ABORT("Failed to remove tls domain"); + } + vespalib::rmdir(removedDir, true); } void ProtonDiskLayout::initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) { - (void) docTypeNames; + vespalib::string documentsDir(getDocumentsDir(_baseDir)); + DocumentDBDirScan dirs; + scanDir(documentsDir, dirs); + for (const auto &dir : dirs) { + if (dir.second.removed) { + // Complete interrupted removal + if (dir.second.normal) { + vespalib::string name(dir.first.toString()); + vespalib::string normalDir(documentsDir + "/" + name); + vespalib::rmdir(normalDir, true); + } + remove(dir.first); + } else if (docTypeNames.count(dir.first) == 0) { + // Remove unused directory + remove(dir.first); + } + } } } // namespace proton |