diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-07-23 10:53:10 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2023-07-25 07:57:52 +0000 |
commit | 59192dd1d07356497ea311b7f0d6a3b50f726f13 (patch) | |
tree | 5f9492d5ce9dd1e4b0125f7d6318127c30a964d6 /searchlib | |
parent | 78a211072a21ec5f368b99bce19c1b703d98152d (diff) |
Prefer std::filesystem::exists over FastOS_StatInfo
Diffstat (limited to 'searchlib')
10 files changed, 77 insertions, 111 deletions
diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index a78dbabe4e3..547a7f3ab53 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - #include <vespa/searchlib/attribute/address_space_components.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> @@ -40,6 +39,7 @@ using search::attribute::BasicType; using search::attribute::IAttributeVector; using vespalib::stringref; using vespalib::string; +namespace fs = std::filesystem; namespace { @@ -98,14 +98,10 @@ expectZero(const string &b) uint64_t statSize(const string &fileName) { - FastOS_StatInfo statInfo; - bool stat_result = true; - EXPECT_TRUE(FastOS_File::Stat(fileName.c_str(), &statInfo)) << (stat_result = false, ""); - if (stat_result) { - return statInfo._size; - } else { - return 0u; - } + std::error_code ec; + uint64_t sz = fs::file_size(fs::path(fileName), ec); + EXPECT_FALSE(ec); + return sz; } uint64_t @@ -698,15 +694,12 @@ AttributeTest::testMemorySaver(const AttributePtr & a) auto b = createAttribute(replace_suffix(*a, "2ms"), a->getConfig()); AttributeMemorySaveTarget saveTarget; EXPECT_TRUE(a->save(saveTarget, b->getBaseFileName())); - FastOS_StatInfo statInfo; vespalib::string datFile = vespalib::make_string("%s.dat", b->getBaseFileName().c_str()); - EXPECT_TRUE(!FastOS_File::Stat(datFile.c_str(), &statInfo)); - EXPECT_TRUE(saveTarget.writeToFile(TuneFileAttributes(), - DummyFileHeaderContext())); - EXPECT_TRUE(FastOS_File::Stat(datFile.c_str(), &statInfo)); + EXPECT_FALSE(fs::exists(fs::path(datFile))); + EXPECT_TRUE(saveTarget.writeToFile(TuneFileAttributes(), DummyFileHeaderContext())); + EXPECT_TRUE(fs::exists(fs::path(datFile))); EXPECT_TRUE(b->load()); - compare<VectorType, BufferType> - (*(static_cast<VectorType *>(a.get())), *(static_cast<VectorType *>(b.get()))); + compare<VectorType, BufferType>(*(static_cast<VectorType *>(a.get())), *(static_cast<VectorType *>(b.get()))); } void @@ -2353,10 +2346,10 @@ AttributeTest::test_paged_attribute(const vespalib::string& name, const vespalib if (failed) { return 0; } - auto size1 = std::filesystem::file_size(std::filesystem::path(swapfile)); + auto size1 = fs::file_size(fs::path(swapfile)); // Grow mapping from lid to value or multivalue index addClearedDocs(av, lid_mapping_size); - auto size2 = std::filesystem::file_size(std::filesystem::path(swapfile)); + auto size2 = fs::file_size(fs::path(swapfile)); auto size3 = size2; EXPECT_LT(size1, size2); if (cfg.collectionType().isMultiValue()) { @@ -2368,7 +2361,7 @@ AttributeTest::test_paged_attribute(const vespalib::string& name, const vespalib } av->commit(); } - size3 = std::filesystem::file_size(std::filesystem::path(swapfile)); + size3 = fs::file_size(fs::path(swapfile)); EXPECT_LT(size2, size3); result += 2; } @@ -2386,7 +2379,7 @@ AttributeTest::test_paged_attribute(const vespalib::string& name, const vespalib } av->commit(); } - auto size4 = std::filesystem::file_size(std::filesystem::path(swapfile)); + auto size4 = fs::file_size(fs::path(swapfile)); EXPECT_LT(size3, size4); result += 4; } @@ -2416,7 +2409,7 @@ AttributeTest::test_paged_attributes() cfg5.setPaged(true); EXPECT_EQ(1, test_paged_attribute("std-bool-sv-paged", basedir + "/4.std-bool-sv-paged/swapfile", cfg5)); vespalib::alloc::MmapFileAllocatorFactory::instance().setup(""); - std::filesystem::remove_all(std::filesystem::path(basedir)); + fs::remove_all(fs::path(basedir)); } void testNamePrefix() { @@ -2576,17 +2569,17 @@ TEST_F(AttributeTest, paged_attributes) void deleteDataDirs() { - std::filesystem::remove_all(std::filesystem::path(tmpDir)); - std::filesystem::remove_all(std::filesystem::path(clsDir)); - std::filesystem::remove_all(std::filesystem::path(asuDir)); + fs::remove_all(fs::path(tmpDir)); + fs::remove_all(fs::path(clsDir)); + fs::remove_all(fs::path(asuDir)); } void createDataDirs() { - std::filesystem::create_directories(std::filesystem::path(tmpDir)); - std::filesystem::create_directories(std::filesystem::path(clsDir)); - std::filesystem::create_directories(std::filesystem::path(asuDir)); + fs::create_directories(fs::path(tmpDir)); + fs::create_directories(fs::path(clsDir)); + fs::create_directories(fs::path(asuDir)); } int diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index f4ab447ed51..56f877a7546 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -25,6 +25,7 @@ #include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/vespalib/util/size_literals.h> #include <thread> +#include <filesystem> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.attributevector"); @@ -39,6 +40,9 @@ using search::common::FileHeaderContext; using search::index::DummyFileHeaderContext; using search::queryeval::SearchIterator; using namespace vespalib::make_string_short; +namespace fs = std::filesystem; + +namespace search { namespace { @@ -47,12 +51,6 @@ const vespalib::string dataTypeTag = "datatype"; const vespalib::string collectionTypeTag = "collectiontype"; const vespalib::string docIdLimitTag = "docIdLimit"; -} - -namespace search { - -namespace { - bool allow_paged(const search::attribute::Config& config) { @@ -78,6 +76,11 @@ make_memory_allocator(const vespalib::string& name, const search::attribute::Con return {}; } +bool +exists(vespalib::stringref name) { + return fs::exists(fs::path(name)); +} + } AttributeVector::AttributeVector(vespalib::stringref baseFileName, const Config &c) @@ -296,30 +299,24 @@ AttributeVector::createAttributeHeader(vespalib::stringref fileName) const { getVersion()); } -void AttributeVector::onSave(IAttributeSaveTarget &) +void +AttributeVector::onSave(IAttributeSaveTarget &) { LOG_ABORT("should not be reached"); } bool AttributeVector::hasLoadData() const { - FastOS_StatInfo statInfo; - if (!FastOS_File::Stat(fmt("%s.dat", getBaseFileName().c_str()).c_str(), &statInfo)) { + if (!exists(getBaseFileName() + ".dat")) { return false; } - if (hasMultiValue() && - !FastOS_File::Stat(fmt("%s.idx", getBaseFileName().c_str()).c_str(), &statInfo)) - { + if (hasMultiValue() && !exists(getBaseFileName() + ".idx")) { return false; } - if (hasWeightedSetType() && - !FastOS_File::Stat(fmt("%s.weight", getBaseFileName().c_str()).c_str(), &statInfo)) - { + if (hasWeightedSetType() && !exists(getBaseFileName() + ".weight")) { return false; } - if (isEnumeratedSaveFormat() && - !FastOS_File::Stat(fmt("%s.udat", getBaseFileName().c_str()).c_str(), &statInfo)) - { + if (isEnumeratedSaveFormat() && !exists(getBaseFileName() + ".udat")) { return false; } return true; diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp index 20a5a76905f..29cf068bb31 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp @@ -6,6 +6,7 @@ #include "pagedict4file.h" #include "field_length_scanner.h" #include <vespa/vespalib/util/error.h> +#include <filesystem> #include <vespa/log/log.h> LOG_SETUP(".diskindex.fieldreader"); @@ -27,6 +28,7 @@ using search::index::Schema; using search::index::SchemaUtil; using search::bitcompression::PosOccFieldParams; using search::bitcompression::PosOccFieldsParams; +namespace fs = std::filesystem; namespace search::diskindex { @@ -124,11 +126,9 @@ FieldReader::open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead) { vespalib::string name = prefix + "posocc.dat.compressed"; - FastOS_StatInfo statInfo; - bool statres = FastOS_File::Stat(name.c_str(), &statInfo); - if (!statres) { - LOG(error, "Could not stat compressed posocc file %s: %s", name.c_str(), getLastErrorString().c_str()); + if (!fs::exists(fs::path(name))) { + LOG(error, "Compressed posocc file %s does not exist: %s", name.c_str(), getLastErrorString().c_str()); return false; } diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 05f3413cdd8..d3553ad003f 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -12,10 +12,13 @@ #include <vespa/vespalib/util/size_literals.h> #include <thread> #include <cassert> +#include <filesystem> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docstore.logdatastore"); +namespace fs = std::filesystem; + namespace search { namespace { @@ -664,14 +667,8 @@ namespace { vespalib::string lsSingleFile(const vespalib::string & fileName) { - vespalib::string s; - FastOS_StatInfo stat; - if ( FastOS_File::Stat(fileName.c_str(), &stat)) { - s += make_string("%s %20" PRIu64 " %12" PRId64, fileName.c_str(), vespalib::count_ns(stat._modifiedTime.time_since_epoch()), stat._size); - } else { - s = make_string("%s 'stat' FAILED !!", fileName.c_str()); - } - return s; + fs::path path(fileName); + return make_string("%s %20" PRIu64 " %12" PRId64, fileName.c_str(), vespalib::count_ns(fs::last_write_time(path).time_since_epoch()), fs::file_size(path)); } } @@ -720,46 +717,34 @@ hasNonHeaderData(const vespalib::string &name) void LogDataStore::verifyModificationTime(const NameIdSet & partList) { - FastOS_StatInfo prevDatStat; - FastOS_StatInfo prevIdxStat; NameId nameId(*partList.begin()); vespalib::string datName(createDatFileName(nameId)); vespalib::string idxName(createIdxFileName(nameId)); - if ( ! FastOS_File::Stat(datName.c_str(), &prevDatStat)) { - throw runtime_error(make_string("Failed to Stat '%s'\nDirectory =\n%s", datName.c_str(), ls(partList).c_str())); - } - if ( ! FastOS_File::Stat(idxName.c_str(), &prevIdxStat)) { - throw runtime_error(make_string("Failed to Stat '%s'\nDirectory =\n%s", idxName.c_str(), ls(partList).c_str())); - } + vespalib::file_time prevDatTime = fs::last_write_time(fs::path(datName)); + vespalib::file_time prevIdxTime = fs::last_write_time(fs::path(idxName));; for (auto it(++partList.begin()), mt(partList.end()); it != mt; ++it) { vespalib::string prevDatNam(datName); vespalib::string prevIdxNam(idxName); - FastOS_StatInfo datStat; - FastOS_StatInfo idxStat; nameId = *it; datName = createDatFileName(nameId); idxName = createIdxFileName(nameId); - if ( ! FastOS_File::Stat(datName.c_str(), &datStat)) { - throw runtime_error(make_string("Failed to Stat '%s'\nDirectory =\n%s", datName.c_str(), ls(partList).c_str())); - } - if ( ! FastOS_File::Stat(idxName.c_str(), &idxStat)) { - throw runtime_error(make_string("Failed to Stat '%s'\nDirectory =\n%s", idxName.c_str(), ls(partList).c_str())); - } + vespalib::file_time datTime = fs::last_write_time(fs::path(datName)); + vespalib::file_time idxTime = fs::last_write_time(fs::path(idxName));; ns_log::Logger::LogLevel logLevel = ns_log::Logger::debug; - if ((datStat._modifiedTime < prevDatStat._modifiedTime) && hasNonHeaderData(datName)) { + if ((datTime < prevDatTime) && hasNonHeaderData(datName)) { VLOG(logLevel, "Older file '%s' is newer (%s) than file '%s' (%s)\nDirectory =\n%s", - prevDatNam.c_str(), to_string(prevDatStat._modifiedTime).c_str(), - datName.c_str(), to_string(datStat._modifiedTime).c_str(), + prevDatNam.c_str(), to_string(prevDatTime).c_str(), + datName.c_str(), to_string(datTime).c_str(), ls(partList).c_str()); } - if ((idxStat._modifiedTime < prevIdxStat._modifiedTime) && hasNonHeaderData(idxName)) { + if ((idxTime < prevIdxTime) && hasNonHeaderData(idxName)) { VLOG(logLevel, "Older file '%s' is newer (%s) than file '%s' (%s)\nDirectory =\n%s", - prevIdxNam.c_str(), to_string(prevIdxStat._modifiedTime).c_str(), - idxName.c_str(), to_string(idxStat._modifiedTime).c_str(), + prevIdxNam.c_str(), to_string(prevIdxTime).c_str(), + idxName.c_str(), to_string(idxTime).c_str(), ls(partList).c_str()); } - prevDatStat = datStat; - prevIdxStat = idxStat; + prevDatTime = datTime; + prevIdxTime = idxTime; } } diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.cpp b/searchlib/src/vespa/searchlib/transactionlog/common.cpp index d4192fe0beb..734223743ae 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/common.cpp @@ -2,17 +2,17 @@ #include "common.h" #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fastos/file.h> #include <filesystem> #include <stdexcept> #include <system_error> -namespace search::transactionlog { - using vespalib::nbostream; using vespalib::nbostream_longlivedbuf; using vespalib::make_string_short::fmt; using std::runtime_error; +namespace fs = std::filesystem; + +namespace search::transactionlog { namespace { @@ -26,20 +26,14 @@ throwRangeError(SerialNum prev, SerialNum next) { } int -makeDirectory(const char * dir) +makeDirectory(vespalib::stringref dir) { - int retval(-1); - - FastOS_StatInfo st; - if ( FastOS_File::Stat(dir, &st) ) { - retval = st._isDirectory ? 0 : -2; - } else { - std::error_code ec; - std::filesystem::create_directory(std::filesystem::path(dir), ec); - retval = (!ec) ? 0 : -3; + if ( fs::exists(fs::path(dir)) ) { + return fs::is_directory(fs::path(dir)) ? 0 : -2; } - - return retval; + std::error_code ec; + fs::create_directory(fs::path(dir), ec); + return (!ec) ? 0 : -3; } int64_t diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.h b/searchlib/src/vespa/searchlib/transactionlog/common.h index 87150f2cfa9..d1d91cd8a4b 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.h +++ b/searchlib/src/vespa/searchlib/transactionlog/common.h @@ -90,7 +90,7 @@ private: vespalib::nbostream_longlivedbuf _buf; }; -int makeDirectory(const char * dir); +int makeDirectory(vespalib::stringref dir); class Writer { public: diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 49fa7041533..6d8cd5b206c 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -61,11 +61,11 @@ Domain::Domain(const string &domainName, const string & baseDir, vespalib::Execu _markedDeleted(false) { assert(_config.getEncoding().getCompression() != Encoding::Compression::none); - int retval = makeDirectory(_baseDir.c_str()); + int retval = makeDirectory(_baseDir); if (retval != 0) { throw runtime_error(fmt("Failed creating basedirectory %s r(%d), e(%d)", _baseDir.c_str(), retval, errno)); } - retval = makeDirectory(dir().c_str()); + retval = makeDirectory(dir()); if (retval != 0) { throw runtime_error(fmt("Failed creating domaindir %s r(%d), e(%d)", dir().c_str(), retval, errno)); } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainconfig.h b/searchlib/src/vespa/searchlib/transactionlog/domainconfig.h index 7701896fa92..fbb25a90130 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainconfig.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domainconfig.h @@ -35,10 +35,10 @@ struct PartInfo { size_t byteSize; vespalib::string file; PartInfo(SerialNumRange range_in, size_t numEntries_in, size_t byteSize_in, vespalib::stringref file_in) - : range(range_in), - numEntries(numEntries_in), - byteSize(byteSize_in), - file(file_in) + : range(range_in), + numEntries(numEntries_in), + byteSize(byteSize_in), + file(file_in) {} }; diff --git a/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp b/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp index f49cef72172..838bdeb6f2e 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp @@ -5,11 +5,12 @@ #include "domain.h" #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/time.h> -#include <vespa/fastos/file.h> +#include <filesystem> using vespalib::slime::Inserter; using vespalib::slime::Cursor; +namespace fs = std::filesystem; namespace search::transactionlog { @@ -34,11 +35,7 @@ struct DomainExplorer : vespalib::StateExplorer { part.setLong("numEntries", part_in.numEntries); part.setLong("byteSize", part_in.byteSize); part.setString("file", part_in.file); - { - FastOS_StatInfo stat_info; - FastOS_File::Stat(part_in.file.c_str(), &stat_info); - part.setString("lastModified", vespalib::to_string(stat_info._modifiedTime)); - } + part.setString("lastModified", vespalib::to_string(fs::last_write_time(fs::path(part_in.file)))); } } } diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index ac9c6318fb5..24f12c644a6 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -107,8 +107,8 @@ TransLogServer::TransLogServer(FNET_Transport & transport, const vespalib::strin _closed(false) { int retval(0); - if ((retval = makeDirectory(_baseDir.c_str())) == 0) { - if ((retval = makeDirectory(dir().c_str())) == 0) { + if ((retval = makeDirectory(_baseDir)) == 0) { + if ((retval = makeDirectory(dir())) == 0) { std::ifstream domainDir(domainList().c_str()); while (domainDir.good() && !domainDir.eof()) { vespalib::string domainName; |