Merge pull request #6490 from vespa-engine/toregge/remove-unused-document-db-dirs

Remove unused document db dirs

5 files changed, 278 insertions, 3 deletions

diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt
index 89e6493cfbc..9d996d96dc7 100644
--- a/searchcore/CMakeLists.txt
+++ b/searchcore/CMakeLists.txt
@@ -129,6 +129,7 @@ vespa_define_module(
     src/tests/proton/proton
     src/tests/proton/proton_config_fetcher
     src/tests/proton/proton_configurer
+    src/tests/proton/proton_disk_layout
     src/tests/proton/reference/gid_to_lid_change_handler
     src/tests/proton/reference/gid_to_lid_change_listener
     src/tests/proton/reference/gid_to_lid_change_registrator
diff --git a/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt b/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt
new file mode 100644
index 00000000000..f63fa21a954
--- /dev/null
+++ b/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_proton_disk_layout_test_app TEST
+    SOURCES
+    proton_disk_layout_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_proton_disk_layout_test_app COMMAND searchcore_proton_disk_layout_test_app)
diff --git a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp
new file mode 100644
index 00000000000..edb4250ce76
--- /dev/null
+++ b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp
@@ -0,0 +1,178 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchcore/proton/server/proton_disk_layout.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/transactionlog/translogserver.h>
+#include <vespa/searchlib/transactionlog/translogclient.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+using search::index::DummyFileHeaderContext;
+using search::transactionlog::TransLogClient;
+using search::transactionlog::TransLogServer;
+using proton::DocTypeName;
+using proton::ProtonDiskLayout;
+
+static constexpr unsigned int tlsPort = 9018;
+
+static const vespalib::string baseDir("testdb");
+static const vespalib::string documentsDir(baseDir + "/documents");
+
+struct FixtureBase
+{
+    FixtureBase() { vespalib::rmdir(baseDir, true); }
+    ~FixtureBase() { vespalib::rmdir(baseDir, true); }
+};
+
+struct DiskLayoutFixture {
+    DummyFileHeaderContext  _fileHeaderContext;
+    TransLogServer          _tls;
+    vespalib::string        _tlsSpec;
+    ProtonDiskLayout        _diskLayout;
+
+    DiskLayoutFixture();
+    ~DiskLayoutFixture();
+
+    void createDirs(const std::set<vespalib::string> &dirs) {
+        for (const auto &dir : dirs) {
+            vespalib::mkdir(documentsDir + "/" + dir, false);
+        }
+    }
+    void createDomains(const std::set<vespalib::string> &domains) {
+        TransLogClient tlc(_tlsSpec);
+        for (const auto &domain : domains) {
+            ASSERT_TRUE(tlc.create(domain));
+        }
+    }
+
+    std::set<vespalib::string> listDomains() {
+        std::vector<vespalib::string> domainVector;
+        TransLogClient tlc(_tlsSpec);
+        ASSERT_TRUE(tlc.listDomains(domainVector));
+        std::set<vespalib::string> domains;
+        for (const auto &domain : domainVector) {
+            domains.emplace(domain);
+        }
+        return domains;
+    }
+
+    std::set<vespalib::string> listDirs() {
+        std::set<vespalib::string> dirs;
+        auto names = vespalib::listDirectory(documentsDir);
+        for (const auto &name : names) {
+            if (vespalib::isDirectory(documentsDir + "/" + name)) {
+                dirs.emplace(name);
+            }
+        }
+        return dirs;
+    }
+
+    void initAndPruneUnused(const std::set<vespalib::string> names)
+    {
+        std::set<DocTypeName> docTypeNames;
+        for (const auto &name: names) {
+            docTypeNames.emplace(name);
+        }
+        _diskLayout.initAndPruneUnused(docTypeNames);
+    }
+
+    void assertDirs(const std::set<vespalib::string> &expDirs) {
+        EXPECT_EQUAL(expDirs, listDirs());
+    }
+
+    void assertDomains(const std::set<vespalib::string> &expDomains)
+    {
+        EXPECT_EQUAL(expDomains, listDomains());
+    }
+};
+
+DiskLayoutFixture::DiskLayoutFixture()
+    : _fileHeaderContext(),
+      _tls("tls", tlsPort, baseDir, _fileHeaderContext),
+      _tlsSpec(vespalib::make_string("tcp/localhost:%u", tlsPort)),
+      _diskLayout(baseDir, _tlsSpec)
+{
+}
+
+DiskLayoutFixture::~DiskLayoutFixture() = default;
+
+struct Fixture : public FixtureBase, public DiskLayoutFixture
+{
+    Fixture()
+        : FixtureBase(),
+          DiskLayoutFixture()
+    {
+    }
+};
+
+TEST_F("require that empty config is ok", Fixture) {
+    TEST_DO(f.assertDirs({}));
+    TEST_DO(f.assertDomains({}));
+}
+
+TEST_F("require that disk layout is preserved", FixtureBase)
+{
+    {
+        DiskLayoutFixture diskLayout;
+        diskLayout.createDirs({"foo", "bar"});
+        diskLayout.createDomains({"bar", "baz"});
+    }
+    {
+        DiskLayoutFixture diskLayout;
+        TEST_DO(diskLayout.assertDirs({"foo", "bar"}));
+        TEST_DO(diskLayout.assertDomains({"bar", "baz"}));
+    }
+}
+
+TEST_F("require that used dir is preserved", Fixture)
+{
+    f.createDirs({"foo"});
+    f.createDomains({"foo"});
+    f.initAndPruneUnused({"foo"});
+    TEST_DO(f.assertDirs({"foo"}));
+    TEST_DO(f.assertDomains({"foo"}));
+}
+
+TEST_F("require that unused dir is removed", Fixture)
+{
+    f.createDirs({"foo"});
+    f.createDomains({"foo"});
+    f.initAndPruneUnused({"bar"});
+    TEST_DO(f.assertDirs({}));
+    TEST_DO(f.assertDomains({}));
+}
+
+TEST_F("require that interrupted remove is completed", Fixture)
+{
+    f.createDirs({"foo.removed"});
+    f.createDomains({"foo"});
+    f.initAndPruneUnused({"foo"});
+    TEST_DO(f.assertDirs({}));
+    TEST_DO(f.assertDomains({}));
+}
+
+TEST_F("require that early interrupted remove is completed", Fixture)
+{
+    f.createDirs({"foo", "foo.removed"});
+    f.createDomains({"foo"});
+    f.initAndPruneUnused({"foo"});
+    TEST_DO(f.assertDirs({}));
+    TEST_DO(f.assertDomains({}));
+}
+
+TEST_F("require that live document db dir remove works", Fixture)
+{
+    f.createDirs({"foo"});
+    f.createDomains({"foo"});
+    f.initAndPruneUnused({"foo"});
+    TEST_DO(f.assertDirs({"foo"}));
+    TEST_DO(f.assertDomains({"foo"}));
+    f._diskLayout.remove(DocTypeName("foo"));
+    TEST_DO(f.assertDirs({}));
+    TEST_DO(f.assertDomains({}));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp
index 00a7e9b9140..c2624719d81 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp
+++ b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp
@@ -44,6 +44,7 @@ TransLogServerMetrics::considerRemoveDomains(const DomainStats &stats)
     for (auto itr = _domainMetrics.begin(); itr != _domainMetrics.end(); ) {
         const vespalib::string &documentType = itr->first;
         if (stats.find(documentType) == stats.end()) {
+            _parent->unregisterMetric(*itr->second);
             itr = _domainMetrics.erase(itr);
         } else {
             ++itr;
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp
index 1bc24d4a0e3..ab4a0576ab0 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp
@@ -3,15 +3,75 @@
 #include "proton_disk_layout.h"
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/fastos/file.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchlib/transactionlog/translogclient.h>
 #include <cassert>
 
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.proton_disk_layout");
+
+using search::transactionlog::TransLogClient;
+
 namespace proton {
 
+namespace {
+
+struct DocumentDBDirMeta
+{
+    bool normal;
+    bool removed;
+
+    DocumentDBDirMeta()
+        : normal(false),
+          removed(false)
+    {
+    }
+};
+
+using DocumentDBDirScan = std::map<DocTypeName, DocumentDBDirMeta>;
+
+vespalib::string getDocumentsDir(const vespalib::string &baseDir)
+{
+    return baseDir + "/documents";
+}
+
+vespalib::string removedSuffix(".removed");
+
+vespalib::string getNormalName(const vespalib::string removedName) {
+    return removedName.substr(0, removedName.size() - removedSuffix.size());
+}
+
+vespalib::string getRemovedName(const vespalib::string &normalName)
+{
+    return normalName + removedSuffix;
+}
+
+bool isRemovedName(const vespalib::string &dirName)
+{
+    return dirName.size() > removedSuffix.size() && dirName.substr(dirName.size() - removedSuffix.size()) == removedSuffix;
+}
+
+void scanDir(const vespalib::string documentsDir, DocumentDBDirScan &dirs)
+{
+    auto names = vespalib::listDirectory(documentsDir);
+    for (const auto &name : names) {
+        if (vespalib::isDirectory(documentsDir + "/" + name)) {
+            if (isRemovedName(name)) {
+                dirs[DocTypeName(getNormalName(name))].removed = true;
+            } else {
+                dirs[DocTypeName(name)].normal = true;
+            }
+        }
+    }
+}
+
+}
+
 ProtonDiskLayout::ProtonDiskLayout(const vespalib::string &baseDir, const vespalib::string &tlsSpec)
     : _baseDir(baseDir),
       _tlsSpec(tlsSpec)
 {
-    vespalib::mkdir(_baseDir + "/documents", true);
+    vespalib::mkdir(getDocumentsDir(_baseDir), true);
 }
 
 ProtonDiskLayout::~ProtonDiskLayout() = default;
@@ -19,13 +79,39 @@ ProtonDiskLayout::~ProtonDiskLayout() = default;
 void
 ProtonDiskLayout::remove(const DocTypeName &docTypeName)
 {
-    (void) docTypeName;
+    vespalib::string documentsDir(getDocumentsDir(_baseDir));
+    vespalib::string name(docTypeName.toString());
+    vespalib::string normalDir(documentsDir + "/" + name);
+    vespalib::string removedDir(documentsDir + "/" + getRemovedName(name));
+    vespalib::rename(normalDir, removedDir, false, false);
+    TransLogClient tlc(_tlsSpec);
+    if (!tlc.remove(name)) {
+        LOG(fatal, "Failed to remove tls domain %s", name.c_str());
+        LOG_ABORT("Failed to remove tls domain");
+    }
+    vespalib::rmdir(removedDir, true);
 }
 
 void
 ProtonDiskLayout::initAndPruneUnused(const std::set<DocTypeName> &docTypeNames)
 {
-    (void) docTypeNames;
+    vespalib::string documentsDir(getDocumentsDir(_baseDir));
+    DocumentDBDirScan dirs;
+    scanDir(documentsDir, dirs);
+    for (const auto &dir : dirs) {
+        if (dir.second.removed) {
+            // Complete interrupted removal
+            if (dir.second.normal) {
+                vespalib::string name(dir.first.toString());
+                vespalib::string normalDir(documentsDir + "/" + name);
+                vespalib::rmdir(normalDir, true);
+            }
+            remove(dir.first);
+        } else if (docTypeNames.count(dir.first) == 0) {
+            // Remove unused directory
+            remove(dir.first);
+        }
+    }
 }
 
 } // namespace proton