diff options
Diffstat (limited to 'searchlib')
14 files changed, 36 insertions, 300 deletions
diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp index 30f417c7532..91eb229b271 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp @@ -48,8 +48,8 @@ updateHeader(const vespalib::string &name, uint64_t fileBitSize) h.putTag(Tag("frozen", 1)); h.putTag(Tag("fileBitSize", fileBitSize)); h.rewriteFile(f); - f.Sync(); - f.Close(); + bool sync_ok = f.Sync(); + assert(sync_ok); } /* diff --git a/searchlib/src/vespa/searchlib/attribute/sourceselector.cpp b/searchlib/src/vespa/searchlib/attribute/sourceselector.cpp index b46866e61e2..3bc85783c2f 100644 --- a/searchlib/src/vespa/searchlib/attribute/sourceselector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/sourceselector.cpp @@ -95,7 +95,6 @@ SourceSelector::LoadInfo::load() if (fileHeader.hasTag(docIdLimitTag)) { _header._docIdLimit = fileHeader.getTag(docIdLimitTag).asInteger(); } - file.Close(); } SourceSelector::SourceSelector(Source defaultSource, AttributeVector::SP realSource) : @@ -106,14 +105,14 @@ SourceSelector::SourceSelector(Source defaultSource, AttributeVector::SP realSou SourceSelector::SaveInfo::UP SourceSelector::extractSaveInfo(const vespalib::string & baseFileName) { - return SaveInfo::UP(new SaveInfo(baseFileName, getDefaultSource(), getBaseId(), - getDocIdLimit(), *_realSource)); + return std::make_unique<SaveInfo>(baseFileName, getDefaultSource(), getBaseId(), + getDocIdLimit(), *_realSource); } SourceSelector::LoadInfo::UP SourceSelector::extractLoadInfo(const vespalib::string & baseFileName) { - return LoadInfo::UP(new LoadInfo(baseFileName)); + return std::make_unique<LoadInfo>(baseFileName); } SourceSelector::Histogram SourceSelector::getDistribution() const diff --git a/searchlib/src/vespa/searchlib/common/documentsummary.cpp b/searchlib/src/vespa/searchlib/common/documentsummary.cpp index 8e70ffa7844..55ecc558598 100644 --- a/searchlib/src/vespa/searchlib/common/documentsummary.cpp +++ b/searchlib/src/vespa/searchlib/common/documentsummary.cpp @@ -33,7 +33,6 @@ DocumentSummary::readDocIdLimit(const vespalib::string &dir, uint32_t &count) p = qcntfile.ReadLine(numbuf, sizeof(numbuf)); while (*p >= '0' && *p <= '9') qcnt = qcnt * 10 + *p++ - '0'; - qcntfile.Close(); count = qcnt; return true; } @@ -52,8 +51,14 @@ DocumentSummary::writeDocIdLimit(const vespalib::string &dir, uint32_t count) } qcntfile.addNum(count, 0, ' '); qcntfile.WriteByte('\n'); - qcntfile.Sync(); - qcntfile.Close(); + if ( ! qcntfile.Sync() ) { + LOG(error, "Could not sync %s: %s", qcntname.c_str(), getLastErrorString().c_str()); + return false; + } + if ( ! qcntfile.Close() ) { + LOG(error, "Could not sync %s: %s", qcntname.c_str(), getLastErrorString().c_str()); + return false; + } return true; } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp index 65ce5decc50..d2b02b02a87 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp @@ -23,7 +23,6 @@ readHeader(vespalib::FileHeader &h, Fast_BufferedFile file(32_Ki); file.OpenReadOnly(name.c_str()); h.readFile(file); - file.Close(); } } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp index f85c6c20624..17ad4c5b846 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp @@ -22,7 +22,6 @@ readHeader(vespalib::FileHeader &h, const vespalib::string &name) Fast_BufferedFile file(32_Ki); file.OpenReadOnly(name.c_str()); h.readFile(file); - file.Close(); } } diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp index 96649d27b82..b845d21d027 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp @@ -15,7 +15,6 @@ namespace { vespalib::string myPId("PageDict4P.1"); vespalib::string mySPId("PageDict4SP.1"); vespalib::string mySSId("PageDict4SS.1"); -vespalib::string emptyId; void assertOpenWriteOnly(bool ok, const vespalib::string &fileName) { @@ -265,10 +264,7 @@ PageDict4FileSeqRead::close() _ssReadContext.setFile(nullptr); _spReadContext.setFile(nullptr); _pReadContext.setFile(nullptr); - _ssfile.Close(); - _spfile.Close(); - _pfile.Close(); - return true; + return _ssfile.Close() && _spfile.Close() &&_pfile.Close(); } diff --git a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp index 49e175ba102..cd9cd6c34da 100644 --- a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp @@ -2,7 +2,6 @@ #include "wordnummapper.h" #include <vespa/fastlib/io/bufferedfile.h> -#include <cassert> namespace search::diskindex { @@ -33,7 +32,6 @@ WordNumMapping::readMappingFile(const vespalib::string &name, old2newwordfile.Read(&map[1], static_cast<size_t>(tempfilesize)); - old2newwordfile.Close(); map[0] = noWordNum(); map[tempfileentries + 1] = noWordNumHigh(); } diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index 4441b868b15..544e8d9f262 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -15,7 +15,6 @@ namespace { vespalib::string myId5("Zc.5"); vespalib::string myId4("Zc.4"); -vespalib::string emptyId; vespalib::string interleaved_features("interleaved_features"); } @@ -98,9 +97,8 @@ Zc4PostingSeqRead::close() { auto &readContext = _reader.get_read_context(); readContext.dropComprBuf(); - _file.Close(); readContext.setFile(nullptr); - return true; + return _file.Close(); } diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index b7b72552914..278fe166272 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -101,7 +101,6 @@ FileChunk::FileChunk(FileId fileId, NameId nameId, const vespalib::string & base _diskFootprint += idxFile.GetSize(); _modificationTime = FileKit::getModificationTime(_idxFileName); } else { - dataFile.Close(); throw SummaryException("Failed opening idx file", idxFile, VESPA_STRLOC); } } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 2ca2f15545d..3a24d348676 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -124,7 +124,9 @@ handleReadError(const char *text, FastOS_FileInterface &file, ssize_t len, ssize e += fmt(" Truncate to %" PRId64 " and continue", lastKnownGoodPos); LOG(error, "%s", e.c_str()); FastOS_File truncateFile(file.GetFileName()); - file.Close(); + if ( ! file.Close()) { + e += getError(file); + } if ( truncateFile.OpenWriteOnlyExisting()) { if (truncateFile.SetSize(lastKnownGoodPos)) { if (truncateFile.Close()) { @@ -243,7 +245,6 @@ DomainPart::buildPacketMapping(bool allowTruncate) } currPos = transLog.GetPosition(); } - transLog.Close(); return currPos; } diff --git a/searchlib/src/vespa/searchlib/util/fileutil.cpp b/searchlib/src/vespa/searchlib/util/fileutil.cpp index 5c213d4e0f0..83f3fe3a5fa 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.cpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.cpp @@ -4,7 +4,6 @@ #include "filesizecalculator.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/guard.h> -#include <cassert> #include <fcntl.h> #include <sys/mman.h> #include <sys/stat.h> @@ -19,12 +18,11 @@ using vespalib::GenericHeader; using vespalib::FileDescriptor; using vespalib::getLastErrorString; -namespace search { -namespace fileutil { +namespace search::fileutil { LoadedMmap::LoadedMmap(const vespalib::string &fileName) - : LoadedBuffer(NULL, 0), - _mapBuffer(NULL), + : LoadedBuffer(nullptr, 0), + _mapBuffer(nullptr), _mapSize(0) { FileDescriptor fd(open(fileName.c_str(), O_RDONLY, 0664)); @@ -35,7 +33,7 @@ LoadedMmap::LoadedMmap(const vespalib::string &fileName) uint64_t fileSize = stbuf.st_size; size_t sz = fileSize; if (sz) { - void *tmpBuffer = mmap(NULL, sz, PROT_READ, MAP_PRIVATE, fd.fd(), 0); + void *tmpBuffer = mmap(nullptr, sz, PROT_READ, MAP_PRIVATE, fd.fd(), 0); if (tmpBuffer != MAP_FAILED) { _mapSize = sz; _mapBuffer = tmpBuffer; @@ -78,14 +76,15 @@ LoadedMmap::~LoadedMmap() { } +namespace search { + std::unique_ptr<FastOS_FileInterface> FileUtil::openFile(const vespalib::string &fileName) { - std::unique_ptr<Fast_BufferedFile> file(new Fast_BufferedFile()); + auto file = std::make_unique<Fast_BufferedFile>(); file->EnableDirectIO(); if (!file->OpenReadOnly(fileName.c_str())) { LOG(error, "could not open %s: %s", file->GetFileName(), getLastErrorString().c_str()); - file->Close(); throw IllegalStateException(make_string("Failed opening '%s' for direct IO reading.", file->GetFileName())); } return file; @@ -97,12 +96,11 @@ using fileutil::LoadedMmap; LoadedBuffer::UP FileUtil::loadFile(const vespalib::string &fileName) { - LoadedBuffer::UP data(new LoadedMmap(fileName)); + auto data = std::make_unique<LoadedMmap>(fileName); FastOS_File file(fileName.c_str()); if (!file.OpenReadOnly()) { LOG(error, "could not open %s: %s", file.GetFileName(), getLastErrorString().c_str()); } - file.Close(); return data; } @@ -125,44 +123,6 @@ void FileWriterBase::handleError(ssize_t numRead, size_t wanted) } } -SequentialFileArray::SequentialFileArray(const vespalib::string & fname) : - _backingFile(), - _name(fname) -{ - _backingFile->EnableDirectIO(); -} - -SequentialFileArray::~SequentialFileArray() -{ - close(); -} - -void SequentialFileArray::rewind() -{ - assert(_backingFile->SetPosition(0)); -} - -void SequentialFileArray::close() -{ - _backingFile->Close(); -} - -void SequentialFileArray::erase() -{ - close(); - FastOS_File::Delete(_backingFile->GetFileName()); -} - -void SequentialFileArray::openReadOnly() -{ - _backingFile->ReadOpen(_name.c_str()); -} - -void SequentialFileArray::openWriteOnly() -{ - _backingFile->OpenWriteOnlyTruncate(_name.c_str()); -} - ssize_t FileReaderBase::read(void *buf, size_t sz) { ssize_t numRead = _file.Read(buf, sz); diff --git a/searchlib/src/vespa/searchlib/util/fileutil.h b/searchlib/src/vespa/searchlib/util/fileutil.h index 27761403b96..74096d8b5dd 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.h +++ b/searchlib/src/vespa/searchlib/util/fileutil.h @@ -104,67 +104,6 @@ public: } }; -class SequentialFileArray -{ -public: - SequentialFileArray(const vespalib::string & fname); - virtual ~SequentialFileArray(); - const vespalib::string & getName() const { return _name; } - void rewind(); - void close(); - void erase(); -protected: - void openReadOnly(); - void openWriteOnly(); - std::unique_ptr<Fast_BufferedFile> _backingFile; - vespalib::string _name; -}; - -template <typename T> -class SequentialFileArrayRead : public SequentialFileArray -{ -public: - SequentialFileArrayRead(const vespalib::string & fname); - ~SequentialFileArrayRead(); - T getNext() const { return _fileReader.readHostOrder(); } - bool hasNext() const; - size_t size() const; -private: - mutable FileReader<T> _fileReader; -}; - -template <typename T> -class SequentialFileArrayWrite : public SequentialFileArray -{ -public: - SequentialFileArrayWrite(const vespalib::string & fname); - void push_back(const T & v) { _count++; _fileWriter.write(&v, sizeof(v)); } - size_t size() const { return _count; } - bool empty() const { return _count == 0; } -private: - size_t _count; - FileWriterBase _fileWriter; -}; - -template <typename T, typename S> -class MergeSorter -{ -public: - MergeSorter(const vespalib::string & name, size_t chunkSize); - void push_back(const T & v); - void commit() { sortChunk(); merge(); } - const vespalib::string & getName() const { return _name; } - void rewind() { } -private: - vespalib::string genName(size_t n); - void merge(); - void sortChunk(); - - std::vector<T> _chunk; - size_t _chunkCount; - vespalib::string _name; -}; - template <typename T> class SequentialReadModifyWriteInterface { @@ -199,32 +138,5 @@ private: size_t _wp; }; -template <typename T, typename R, typename W> -class SequentialReaderWriter : public SequentialReadModifyWriteInterface<T> -{ -public: - SequentialReaderWriter(R & reader, W & writer); - ~SequentialReaderWriter(); - virtual const T & read() { return _lastRead; } - virtual void write(const T & v) { _writer.push_back(v); } - virtual bool next() { - bool hasMore(_reader.hasNext()); - if (hasMore) { - _lastRead = _reader.getNext(); - } - return hasMore; - } - virtual size_t size() const { return _reader.size(); } - virtual void rewind() { - _reader.rewind(); - next(); - _writer.rewind(); - } -private: - T _lastRead; - R & _reader; - W & _writer; -}; - } diff --git a/searchlib/src/vespa/searchlib/util/fileutil.hpp b/searchlib/src/vespa/searchlib/util/fileutil.hpp index eb0d62b195e..98e54581ac7 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.hpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.hpp @@ -7,122 +7,6 @@ namespace search { template <typename T> -SequentialFileArrayRead<T>::SequentialFileArrayRead(const vespalib::string & fname) : - SequentialFileArray(fname), - _fileReader(std::make_unique<Fast_BufferedFile>(_backingFile)) -{ - openReadOnly(); -} - -template <typename T> -bool -SequentialFileArrayRead<T>::hasNext() const { - return _backingFile->BytesLeft() >= sizeof(T); - -} - -template <typename T> -size_t SequentialFileArrayRead<T>::size() const -{ - return _backingFile->GetSize()/sizeof(T); -} - -template <typename T> -SequentialFileArrayWrite<T>::SequentialFileArrayWrite(const vespalib::string & fname) : - SequentialFileArray(fname), - _count(0), - _fileWriter(_backingFile) -{ - openWriteOnly(); -} - -template <typename T, typename S> -MergeSorter<T, S>::MergeSorter(const vespalib::string & name, size_t chunkSize) : - _chunk(), - _chunkCount(0), - _name(name + ".sorted") -{ - _chunk.reserve(chunkSize); -} - -template <typename T, typename S> -void MergeSorter<T, S>::push_back(const T & v) -{ - if (_chunk.size() < _chunk.capacity()) { - _chunk.push_back(v); - if (_chunk.size() == _chunk.capacity()) { - sortChunk(); - } - } -} - -template <typename T, typename S> -vespalib::string MergeSorter<T, S>::genName(size_t n) -{ - char tmp[32]; - sprintf(tmp, ".%zd", n); - vespalib::string fname(_name); - fname += tmp; - return fname; -} - -template <typename T, typename S> -void MergeSorter<T, S>::merge() -{ - S sorter; - std::vector< SequentialFileArrayRead<T> *> fileParts; - size_t count(0); - for(size_t i(0); i < _chunkCount; i++) { - std::unique_ptr< SequentialFileArrayRead<T> > part(new SequentialFileArrayRead<T>(genName(i))); - size_t sz = part->size(); - if (sz > 0) { - fileParts.push_back(part.release()); - } else { - part->erase(); - } - count += sz; - } - - std::vector<T> cachedValue; - for(size_t i(0), m(fileParts.size()); i < m; i++) { - cachedValue.push_back(fileParts[i]->getNext()); - } - SequentialFileArrayWrite<T> merged(_name); - for(size_t j(0); j < count; j++) { - size_t firstIndex(0); - for(size_t i(1), m(cachedValue.size()); i < m; i++) { - if (sorter.cmp(cachedValue[i], cachedValue[firstIndex])) { - firstIndex = i; - } - } - merged.push_back(cachedValue[firstIndex]); - if ( ! fileParts[firstIndex]->hasNext() ) { - fileParts[firstIndex]->erase(); - delete fileParts[firstIndex]; - fileParts.erase(fileParts.begin()+firstIndex); - cachedValue.erase(cachedValue.begin()+firstIndex); - } else { - cachedValue[firstIndex] = fileParts[firstIndex]->getNext(); - } - } -} - -template <typename T, typename S> -void MergeSorter<T, S>::sortChunk() -{ - S sorter; - sorter.sort(&_chunk[0], _chunk.size()); - FastOS_File chunkFile(genName(_chunkCount).c_str()); - chunkFile.EnableDirectIO(); - if (chunkFile.OpenWriteOnlyTruncate()) { - chunkFile.CheckedWrite(&_chunk[0], _chunk.size()*sizeof(_chunk[0])); - } - chunkFile.Close(); - _chunkCount++; - _chunk.clear(); -} - -template <typename T> SequentialReadModifyWriteVector<T>::SequentialReadModifyWriteVector() : Vector(), _rp(0), @@ -137,17 +21,6 @@ SequentialReadModifyWriteVector<T>::SequentialReadModifyWriteVector(size_t sz) { } template <typename T> -SequentialReadModifyWriteVector<T>::~SequentialReadModifyWriteVector() { } - -template <typename T, typename R, typename W> -SequentialReaderWriter<T, R, W>::SequentialReaderWriter(R & reader, W & writer) - : _reader(reader), - _writer(writer) -{ - next(); -} - -template <typename T, typename R, typename W> -SequentialReaderWriter<T, R, W>::~SequentialReaderWriter() { } +SequentialReadModifyWriteVector<T>::~SequentialReadModifyWriteVector() = default; } diff --git a/searchlib/src/vespa/searchlib/util/stringenum.cpp b/searchlib/src/vespa/searchlib/util/stringenum.cpp index 01c4d4e785e..9744d39746e 100644 --- a/searchlib/src/vespa/searchlib/util/stringenum.cpp +++ b/searchlib/src/vespa/searchlib/util/stringenum.cpp @@ -13,20 +13,19 @@ namespace search::util { static inline char * StripString(char *str) { - char *first = NULL; // first non-space char - char *last = NULL; // last non-space char + char *last = nullptr; // last non-space char - if (str == NULL) - return NULL; + if (str == nullptr) + return nullptr; for (; *str != '\0' && isspace(*str); str++); - first = str; + char *first = str; for (; *str != '\0'; str++) if (!isspace(*str)) last = str; - if (last != NULL) + if (last != nullptr) *(last + 1) = '\0'; return first; @@ -76,8 +75,7 @@ StringEnum::Save(const char *filename) file.WriteString("\n"); } - file.Close(); - return true; + return file.Sync(); } @@ -100,7 +98,7 @@ StringEnum::Load(const char *filename) entryCnt = 0; pt = StripString(file.ReadLine(line, sizeof(line))); - if (pt == NULL || *pt == '\0') + if (pt == nullptr || *pt == '\0') return false; lineNumber++; @@ -108,7 +106,7 @@ StringEnum::Load(const char *filename) while (!file.Eof()) { pt = StripString(file.ReadLine(line, sizeof(line))); - if (pt == NULL) // end of input ? + if (pt == nullptr) // end of input ? break; lineNumber++; if (*pt == '\0') // empty line ? @@ -121,7 +119,6 @@ StringEnum::Load(const char *filename) entryCnt++; } - file.Close(); if (entries != _numEntries || entries != entryCnt) { Clear(); @@ -162,7 +159,7 @@ const char * StringEnum::Lookup(uint32_t value) const { if (value >= _numEntries) - return NULL; + return nullptr; if (_numEntries > _reverseMap.size()) CreateReverseMapping(); |