From 54741a65578269abf9cd515f4246320162e8713e Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 24 Jun 2022 13:45:01 +0200 Subject: Remove unused code for creating / removing directories and for recursive directory traversal. --- searchlib/src/vespa/searchlib/util/dirtraverse.cpp | 282 +++------------------ searchlib/src/vespa/searchlib/util/dirtraverse.h | 48 +--- 2 files changed, 40 insertions(+), 290 deletions(-) (limited to 'searchlib') diff --git a/searchlib/src/vespa/searchlib/util/dirtraverse.cpp b/searchlib/src/vespa/searchlib/util/dirtraverse.cpp index 07dbc9a247d..c1e8b6b7396 100644 --- a/searchlib/src/vespa/searchlib/util/dirtraverse.cpp +++ b/searchlib/src/vespa/searchlib/util/dirtraverse.cpp @@ -2,275 +2,63 @@ #include "dirtraverse.h" #include -#include -#include -#include +#include +#include namespace search { -extern "C" { -static int cmpname(const void *av, const void *bv) -{ - const DirectoryTraverse::Name *const a = - *(const DirectoryTraverse::Name *const *) av; - const DirectoryTraverse::Name *const b = - *(const DirectoryTraverse::Name *const *) bv; - return a->_name.compare(b->_name.c_str()); -} -} - -DirectoryTraverse::Name::Name(const char *name) - : _name(name), - _next(nullptr) -{ -} -DirectoryTraverse::Name::~Name() = default; - -DirectoryTraverse::Name * -DirectoryTraverse::Name::sort(Name *head, int count) -{ - Name *nl; - Name **names; - int i; - - names = new Name *[count]; - i = 0; - for(nl = head; nl != nullptr; nl = nl->_next) - names[i++] = nl; - assert(i == count); - qsort(names, count, sizeof(Name *), cmpname); - for (i = 0; i < count; i++) { - if (i + 1 < count) - names[i]->_next = names[i + 1]; - else - names[i]->_next = nullptr; - } - head = names[0]; - delete [] names; - return head; -} - - -void -DirectoryTraverse::QueueDir(const char *name) -{ - Name *n = new Name(name); - if (_dirTail == nullptr) - _dirHead = n; - else - _dirTail->_next = n; - _dirTail = n; -} +namespace fs = std::filesystem; +namespace { -void -DirectoryTraverse::PushDir(const char *name) -{ - Name *n = new Name(name); - n->_next = _pdirHead; - _pdirHead = n; -} - - -void -DirectoryTraverse::PushRemoveDir(const char *name) -{ - Name *n = new Name(name); - n->_next = _rdirHead; - _rdirHead = n; -} - - -void -DirectoryTraverse::PushPushedDirs() -{ - Name *n; - while (_pdirHead != nullptr) { - n = _pdirHead; - _pdirHead = n->_next; - n->_next = _dirHead; - _dirHead = n; - if (_dirTail == nullptr) - _dirTail = n; - } -} - - -DirectoryTraverse::Name * -DirectoryTraverse::UnQueueDir() -{ - Name *n; - PushPushedDirs(); - if (_dirHead == nullptr) - return nullptr; - n = _dirHead; - _dirHead = n->_next; - n->_next = nullptr; - if (_dirHead == nullptr) - _dirTail = nullptr; - return n; -} - -DirectoryTraverse::Name * -DirectoryTraverse::UnQueueName() -{ - Name *n; - if (_nameHead == nullptr) - return nullptr; - n = _nameHead; - _nameHead = n->_next; - n->_next = nullptr; - _nameCount--; - return n; -} - - -void -DirectoryTraverse::ScanSingleDir() +uint64_t +try_get_tree_size(const std::string& base_dir) { - assert(_nameHead == nullptr); - assert(_nameCount == 0); - delete _curDir; - _fullDirName.clear(); - _curDir = UnQueueDir(); - if (_curDir == nullptr) - return; - _fullDirName = _baseDir; - if ( ! _curDir->_name.empty()) { - _fullDirName += "/" + _curDir->_name; + fs::path path(base_dir); + std::error_code ec; + fs::recursive_directory_iterator dir_itr(path, fs::directory_options::skip_permission_denied, ec); + if (ec) { + return 0; } - FastOS_DirectoryScan *dirscan = new FastOS_DirectoryScan(_fullDirName.c_str()); - while (dirscan->ReadNext()) { - const char *name = dirscan->GetName(); - if (strcmp(name, ".") == 0 || - strcmp(name, "..") == 0) - continue; - Name *nl = new Name(name); - nl->_next = _nameHead; - _nameHead = nl; - _nameCount++; - } - if (_nameCount > 1) - _nameHead = _nameHead->sort(_nameHead, _nameCount); - delete dirscan; -} - - -bool -DirectoryTraverse::NextName() -{ - delete _curName; - _curName = nullptr; - while (_nameHead == nullptr && (_dirHead != nullptr || _pdirHead != nullptr)) - ScanSingleDir(); - if (_nameHead == nullptr) - return false; - _curName = UnQueueName(); - _fullName = _fullDirName + "/" + _curName->_name; - _relName = _fullName.c_str() + (_baseDir.size() + 1); - return true; -} - - -bool -DirectoryTraverse::NextRemoveDir() -{ - Name *curName; - delete _curName; - _curName = nullptr; - if (_rdirHead == nullptr) - return false; - curName = _rdirHead; - _rdirHead = curName->_next; - _fullName = _baseDir + "/" + curName->_name; - _relName = _fullName.c_str() + _baseDir.size() + 1; - delete curName; - return true; -} - - -bool -DirectoryTraverse::RemoveTree() -{ - FastOS_StatInfo statInfo; - - while (NextName()) { - const char *relname = GetRelName(); - const char *fullname = GetFullName(); - if (FastOS_File::Stat(fullname, &statInfo)) { - if (statInfo._isDirectory) { - PushDir(relname); - PushRemoveDir(relname); - } else { - FastOS_File::Delete(fullname); + uint64_t total_size = 0; + constexpr uint64_t block_size = 4_Ki; + for (const auto &elem : dir_itr) { + if (fs::is_regular_file(elem.path()) && !fs::is_symlink(elem.path())) { + const auto size = elem.file_size(ec); + if (!ec) { + // round up size to file system block size (assumed to be 4 KiB) + auto adj_size = ((size + block_size - 1) / block_size) * block_size; + total_size += adj_size; } } } - while (NextRemoveDir()) { - const char *fullname = GetFullName(); - FastOS_File::RemoveDirectory(fullname); - } - FastOS_File::RemoveDirectory(_baseDir.c_str()); - return true; + return total_size; +} + } uint64_t DirectoryTraverse::GetTreeSize() { - FastOS_StatInfo statInfo; - uint64_t size = 0; - const uint64_t blockSize = 4_Ki; - - while (NextName()) { - const char *relname = GetRelName(); - const char *fullname = GetFullName(); - if (FastOS_File::Stat(fullname, &statInfo)) { - uint64_t adjSize = ((statInfo._size + blockSize - 1) / blockSize) * blockSize; - size += adjSize; - if (statInfo._isDirectory) { - PushDir(relname); - } + // Since try_get_tree_size may throw on concurrent directory + // modifications, immediately retry a bounded number of times if this + // happens. Number of retries chosen randomly by counting fingers. + for (int i = 0; i < 10; ++i) { + try { + return try_get_tree_size(_base_dir); + } catch (const fs::filesystem_error&) { + // Go around for another spin that hopefully won't race. } } - return size; + return 0; } -DirectoryTraverse::DirectoryTraverse(const char *baseDir) - : _baseDir(baseDir), - _nameHead(nullptr), - _nameCount(0), - _dirHead(nullptr), - _dirTail(nullptr), - _pdirHead(nullptr), - _rdirHead(nullptr), - _curDir(nullptr), - _curName(nullptr), - _fullDirName(), - _fullName(), - _relName(nullptr) +DirectoryTraverse::DirectoryTraverse(const std::string& base_dir) + : _base_dir(base_dir) { - QueueDir(""); - ScanSingleDir(); } - -DirectoryTraverse::~DirectoryTraverse() -{ - delete _curDir; - delete _curName; - PushPushedDirs(); - while (_dirHead != nullptr) - delete UnQueueDir(); - while (_nameHead != nullptr) - delete UnQueueName(); - while (_rdirHead != nullptr) { - Name *n; - n = _rdirHead; - _rdirHead = n->_next; - n->_next = nullptr; - delete n; - } -} +DirectoryTraverse::~DirectoryTraverse() = default; } // namespace search diff --git a/searchlib/src/vespa/searchlib/util/dirtraverse.h b/searchlib/src/vespa/searchlib/util/dirtraverse.h index 4a96ad0935d..c26246e2596 100644 --- a/searchlib/src/vespa/searchlib/util/dirtraverse.h +++ b/searchlib/src/vespa/searchlib/util/dirtraverse.h @@ -7,54 +7,16 @@ namespace search { +/* + * Class used to get size of directory tree on disk. + */ class DirectoryTraverse { private: - DirectoryTraverse(const DirectoryTraverse &); - DirectoryTraverse& operator=(const DirectoryTraverse &); - -public: - class Name - { - private: - Name(const Name &); - Name& operator=(const Name &); - - public: - std::string _name; - Name *_next; - explicit Name(const char *name); - ~Name(); - static Name *sort(Name *head, int count); - }; -private: - std::string _baseDir; - Name *_nameHead; - int _nameCount; - Name *_dirHead; - Name *_dirTail; - Name *_pdirHead; - Name *_rdirHead; - Name *_curDir; - Name *_curName; - std::string _fullDirName; - std::string _fullName; - const char *_relName; + std::string _base_dir; public: - const char *GetFullName() const { return _fullName.c_str(); } - const char *GetRelName() const { return _relName; } - void QueueDir(const char *name); - void PushDir(const char *name); - void PushRemoveDir(const char *name); - void PushPushedDirs(); - Name *UnQueueDir(); - Name *UnQueueName(); - void ScanSingleDir(); - bool NextName(); - bool NextRemoveDir(); - bool RemoveTree(); uint64_t GetTreeSize(); // Returns size of directory in bytes - explicit DirectoryTraverse(const char *baseDir); + explicit DirectoryTraverse(const std::string& base_dir); ~DirectoryTraverse(); }; -- cgit v1.2.3