From 184f37836ae30b7c468dd4b1305695939ed92357 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 27 Jan 2022 21:10:24 +0000 Subject: GC unused Load/Save code for StringEnum --- searchlib/src/tests/stringenum/stringenum_test.cpp | 75 +---------------- searchlib/src/vespa/searchlib/util/stringenum.cpp | 94 ---------------------- searchlib/src/vespa/searchlib/util/stringenum.h | 19 ----- 3 files changed, 4 insertions(+), 184 deletions(-) diff --git a/searchlib/src/tests/stringenum/stringenum_test.cpp b/searchlib/src/tests/stringenum/stringenum_test.cpp index 15ec1862338..f5915db6df0 100644 --- a/searchlib/src/tests/stringenum/stringenum_test.cpp +++ b/searchlib/src/tests/stringenum/stringenum_test.cpp @@ -1,43 +1,28 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include -LOG_SETUP("stringenum"); -#include #include - +LOG_SETUP("stringenum"); #include using namespace vespalib; -class MyApp : public vespalib::TestApp -{ -public: - void CheckLookup( search::util::StringEnum *strEnum, const char *str, int value); - int Main() override; - MyApp() {} -}; - - void -MyApp::CheckLookup( search::util::StringEnum *strEnum, const char *str, int value) +CheckLookup( search::util::StringEnum *strEnum, const char *str, int value) { EXPECT_EQUAL(0, strcmp(str, strEnum->Lookup(value))); EXPECT_EQUAL(value, strEnum->Lookup(str)); } -int -MyApp::Main() +TEST("test StringEnum Add and Lookup") { - TEST_INIT("stringenum_test"); search::util::StringEnum enum1; - search::util::StringEnum enum2; // check number of entries EXPECT_EQUAL(enum1.GetNumEntries(), 0u); - EXPECT_EQUAL(enum2.GetNumEntries(), 0u); // check add non-duplicates EXPECT_EQUAL(enum1.Add("zero"), 0); @@ -80,63 +65,11 @@ MyApp::Main() TEST_DO(CheckLookup(&enum1, "nine", 9)); TEST_DO(CheckLookup(&enum1, "ten", 10)); - TEST_FLUSH(); - - // save/load - EXPECT_TRUE(enum1.Save("tmp.enum")); - EXPECT_TRUE(enum2.Load("tmp.enum")); - - // check mapping and reverse mapping - EXPECT_EQUAL(enum2.GetNumEntries(), 11u); - TEST_DO(CheckLookup(&enum2, "zero", 0)); - TEST_DO(CheckLookup(&enum2, "one", 1)); - TEST_DO(CheckLookup(&enum2, "two", 2)); - TEST_DO(CheckLookup(&enum2, "three", 3)); - TEST_DO(CheckLookup(&enum2, "four", 4)); - TEST_DO(CheckLookup(&enum2, "five", 5)); - TEST_DO(CheckLookup(&enum2, "six", 6)); - TEST_DO(CheckLookup(&enum2, "seven", 7)); - TEST_DO(CheckLookup(&enum2, "eight", 8)); - TEST_DO(CheckLookup(&enum2, "nine", 9)); - TEST_DO(CheckLookup(&enum2, "ten", 10)); - - // add garbage - enum2.Add("sfsdffgdfh"); - enum2.Add("sf24dfsgg3"); - enum2.Add("sfwertfgdh"); - enum2.Add("sfewrgtsfh"); - enum2.Add("sfgdsdgdfh"); - - TEST_FLUSH(); - - // reload - EXPECT_TRUE(enum2.Load("tmp.enum")); - - // check garbage lost - EXPECT_EQUAL(enum2.GetNumEntries(), 11u); - EXPECT_EQUAL(-1, enum2.Lookup("sfewrgtsfh")); - // check mapping and reverse mapping - TEST_DO(CheckLookup(&enum2, "zero", 0)); - TEST_DO(CheckLookup(&enum2, "one", 1)); - TEST_DO(CheckLookup(&enum2, "two", 2)); - TEST_DO(CheckLookup(&enum2, "three", 3)); - TEST_DO(CheckLookup(&enum2, "four", 4)); - TEST_DO(CheckLookup(&enum2, "five", 5)); - TEST_DO(CheckLookup(&enum2, "six", 6)); - TEST_DO(CheckLookup(&enum2, "seven", 7)); - TEST_DO(CheckLookup(&enum2, "eight", 8)); - TEST_DO(CheckLookup(&enum2, "nine", 9)); - TEST_DO(CheckLookup(&enum2, "ten", 10)); - // clear enum1.Clear(); - enum2.Clear(); // check number of entries EXPECT_EQUAL(enum1.GetNumEntries(), 0u); - EXPECT_EQUAL(enum2.GetNumEntries(), 0u); - - TEST_DONE(); } -TEST_APPHOOK(MyApp); +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/util/stringenum.cpp b/searchlib/src/vespa/searchlib/util/stringenum.cpp index 9744d39746e..116e400083a 100644 --- a/searchlib/src/vespa/searchlib/util/stringenum.cpp +++ b/searchlib/src/vespa/searchlib/util/stringenum.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "stringenum.h" -#include #include #include @@ -10,27 +9,6 @@ LOG_SETUP(".seachlib.util.stringenum"); namespace search::util { -static inline char * -StripString(char *str) -{ - char *last = nullptr; // last non-space char - - if (str == nullptr) - return nullptr; - - for (; *str != '\0' && isspace(*str); str++); - char *first = str; - - for (; *str != '\0'; str++) - if (!isspace(*str)) - last = str; - - if (last != nullptr) - *(last + 1) = '\0'; - - return first; -} - StringEnum::StringEnum() : _numEntries(0), _mapping(), @@ -55,78 +33,6 @@ StringEnum::CreateReverseMapping() const } } - -bool -StringEnum::Save(const char *filename) -{ - char str[1024]; - - Fast_BufferedFile file; - file.WriteOpen(filename); - if (!file.IsOpened()) - return false; - - file.SetSize(0); - sprintf(str, "%d\n", _numEntries); - file.WriteString(str); - - for (uint32_t i = 0; i < _numEntries; i++) { - file.WriteString(Lookup(i)); - file.WriteString("\n"); - } - - return file.Sync(); -} - - -bool -StringEnum::Load(const char *filename) -{ - char line[1024]; - char *pt; - uint32_t entries; // from first line of file - uint32_t lineNumber; // current line in file - uint32_t entryCnt; // # entries obtained from file - - Clear(); - - Fast_BufferedFile file; - if (!file.OpenReadOnly(filename)) - return false; - - lineNumber = 0; - entryCnt = 0; - - pt = StripString(file.ReadLine(line, sizeof(line))); - if (pt == nullptr || *pt == '\0') - return false; - lineNumber++; - - entries = atoi(pt); - - while (!file.Eof()) { - pt = StripString(file.ReadLine(line, sizeof(line))); - if (pt == nullptr) // end of input ? - break; - lineNumber++; - if (*pt == '\0') // empty line ? - continue; - - uint32_t tmp = _numEntries; - if (static_cast(Add(pt)) != tmp) { - LOG(error, "(%s:%d) duplicate enum entry: %s", filename, lineNumber, pt); - } - entryCnt++; - } - - if (entries != _numEntries - || entries != entryCnt) { - Clear(); - return false; - } - return true; -} - void StringEnum::Clear() { diff --git a/searchlib/src/vespa/searchlib/util/stringenum.h b/searchlib/src/vespa/searchlib/util/stringenum.h index 85f97de48ad..0da79db323a 100644 --- a/searchlib/src/vespa/searchlib/util/stringenum.h +++ b/searchlib/src/vespa/searchlib/util/stringenum.h @@ -80,25 +80,6 @@ public: * @return current number of entries. **/ uint32_t GetNumEntries() const { return _numEntries; } - - - /** - * Save the enumeration currently held by this object to file. - * - * @return success(true)/fail(false). - * @param filename name of save file. - **/ - bool Save(const char *filename); - - - /** - * Load an enumeration from file. The loaded enumeration will - * replace the one currently held by this object. - * - * @return success(true)/fail(false). - * @param filename name of file to load. - **/ - bool Load(const char *filename); }; } -- cgit v1.2.3 From 117cd771441cf1b8cc1f93541f8868e0b1b8e46e Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 27 Jan 2022 23:10:03 +0000 Subject: Hide Fast_BufferedFile better. --- fastlib/src/vespa/fastlib/io/bufferedfile.cpp | 73 +++++++++------------- fastlib/src/vespa/fastlib/io/bufferedfile.h | 29 +++------ searchlib/src/tests/attribute/attribute_test.cpp | 8 +-- .../vespa/searchlib/attribute/attributevector.h | 3 - .../src/vespa/searchlib/attribute/attrvector.hpp | 1 - .../searchlib/attribute/multinumericattribute.hpp | 1 - .../attribute/multinumericenumattribute.hpp | 1 - .../searchlib/attribute/multistringattribute.hpp | 1 - .../attribute/multistringpostattribute.hpp | 1 - .../src/vespa/searchlib/attribute/readerbase.cpp | 1 - .../searchlib/attribute/singlestringattribute.hpp | 1 - .../vespa/searchlib/diskindex/bitvectorfile.cpp | 31 +++++---- .../src/vespa/searchlib/diskindex/bitvectorfile.h | 15 ++--- .../vespa/searchlib/diskindex/bitvectoridxfile.cpp | 16 ++--- .../vespa/searchlib/diskindex/bitvectoridxfile.h | 17 ++--- .../searchlib/diskindex/dictionarywordreader.cpp | 15 +++-- .../searchlib/diskindex/dictionarywordreader.h | 8 +-- .../src/vespa/searchlib/diskindex/zcposting.h | 2 - .../src/vespa/searchlib/tensor/CMakeLists.txt | 1 + .../searchlib/tensor/blob_sequence_reader.cpp | 13 ++++ .../vespa/searchlib/tensor/blob_sequence_reader.h | 6 +- .../searchlib/tensor/direct_tensor_attribute.cpp | 1 - .../tensor/serialized_fast_value_attribute.cpp | 3 - searchlib/src/vespa/searchlib/util/fileutil.cpp | 1 + searchlib/src/vespa/searchlib/util/fileutil.h | 1 - searchlib/src/vespa/searchlib/util/fileutil.hpp | 1 - 26 files changed, 110 insertions(+), 141 deletions(-) create mode 100644 searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.cpp diff --git a/fastlib/src/vespa/fastlib/io/bufferedfile.cpp b/fastlib/src/vespa/fastlib/io/bufferedfile.cpp index d93dbd74ee1..56ffbf40eca 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedfile.cpp +++ b/fastlib/src/vespa/fastlib/io/bufferedfile.cpp @@ -13,7 +13,7 @@ const size_t MIN_ALIGNMENT = 0x1000; } void -Fast_BufferedFile::flushWriteBuf(void) +Fast_BufferedFile::flushWriteBuf() { if (_bufi != buf()) { _file->WriteBuf(buf(), _bufi - buf()); @@ -31,7 +31,7 @@ Fast_BufferedFile::flushWriteBuf(void) } void -Fast_BufferedFile::fillReadBuf(void) +Fast_BufferedFile::fillReadBuf() { size_t toread = std::min(static_cast(_buf.size()), _fileleft); if (toread > 0) { @@ -71,20 +71,14 @@ Fast_BufferedFile::addNum(unsigned int num, int fieldw, char fill) } } -uint64_t -Fast_BufferedFile::BytesLeft(void) const -{ - return _fileleft + (_bufe - _bufi); -} - bool -Fast_BufferedFile::Eof(void) const +Fast_BufferedFile::Eof() const { return _fileleft == 0 && _bufi == _bufe; } int64_t -Fast_BufferedFile::GetSize (void) +Fast_BufferedFile::GetSize() { return _file->GetSize(); } @@ -101,41 +95,41 @@ Fast_BufferedFile::SetSize (int64_t s) } bool -Fast_BufferedFile::IsOpened (void) const +Fast_BufferedFile::IsOpened () const { return _file->IsOpened(); } bool -Fast_BufferedFile::Sync(void) +Fast_BufferedFile::Sync() { Flush(); return _file->Sync(); } time_t -Fast_BufferedFile::GetModificationTime(void) +Fast_BufferedFile::GetModificationTime() { time_t retval = _file->GetModificationTime(); return retval; } void -Fast_BufferedFile::EnableDirectIO(void) +Fast_BufferedFile::EnableDirectIO() { _file->EnableDirectIO(); _directIOEnabled = true; } void -Fast_BufferedFile::EnableSyncWrites(void) +Fast_BufferedFile::EnableSyncWrites() { FastOS_FileInterface::EnableSyncWrites(); _file->EnableSyncWrites(); } int64_t -Fast_BufferedFile::GetPosition(void) +Fast_BufferedFile::GetPosition() { if (_file->IsWriteMode()) { int64_t filePosition = _file->GetPosition(); @@ -147,7 +141,7 @@ Fast_BufferedFile::GetPosition(void) void -Fast_BufferedFile::Flush(void) +Fast_BufferedFile::Flush() { if (_file->IsWriteMode()) { flushWriteBuf(); @@ -202,7 +196,7 @@ Fast_BufferedFile::SetPosition(const int64_t s) } const char * -Fast_BufferedFile::GetFileName(void) const +Fast_BufferedFile::GetFileName() const { return _file ? _file->GetFileName() : ""; } @@ -233,7 +227,7 @@ Fast_BufferedFile::ReadLine(char *line, size_t buflen) continue; } *p++ = *_bufi++; - *p++ = 0; + *p = 0; return line; } } @@ -294,22 +288,11 @@ Fast_BufferedFile::WriteByte(char byte) *_bufi++ = byte; } -int -Fast_BufferedFile::GetByte(void) -{ - if (_bufi < _bufe) - return *reinterpret_cast(_bufi++); - fillReadBuf(); - if (_bufi < _bufe) - return *reinterpret_cast(_bufi++); - return -1; -} - void Fast_BufferedFile::ReadOpenExisting(const char *name) { - Close(); - bool ok = _file->OpenReadOnlyExisting(true, name); + bool ok = Close(); + ok &= _file->OpenReadOnlyExisting(true, name); if (!ok) { fprintf(stderr, "ERROR opening %s for read: %s", _file->GetFileName(), getLastErrorString().c_str()); @@ -325,8 +308,8 @@ Fast_BufferedFile::ReadOpenExisting(const char *name) void Fast_BufferedFile::ReadOpen(const char *name) { - Close(); - bool ok = _file->OpenReadOnly(name); + bool ok = Close(); + ok &= _file->OpenReadOnly(name); if (!ok) { fprintf(stderr, "ERROR opening %s for read: %s", _file->GetFileName(), getLastErrorString().c_str()); @@ -345,8 +328,8 @@ Fast_BufferedFile::ReadOpen(const char *name) void Fast_BufferedFile::WriteOpen(const char *name) { - Close(); - bool ok = _file->OpenWriteOnly(name); + bool ok = Close(); + ok &= _file->OpenWriteOnly(name); if (!ok) { fprintf(stderr, "ERROR opening %s for write: %s", _file->GetFileName(), getLastErrorString().c_str()); @@ -377,7 +360,7 @@ namespace { size_t computeBufLen(size_t buflen) { - size_t bitCount(0); + size_t bitCount; for ( bitCount = 1; buflen >> bitCount; bitCount++); buflen = 1 << (bitCount - 1); @@ -402,21 +385,21 @@ Fast_BufferedFile::Fast_BufferedFile(FastOS_FileInterface *file, size_t bufferSi ResetBuf(); } -Fast_BufferedFile::~Fast_BufferedFile(void) +Fast_BufferedFile::~Fast_BufferedFile() { bool close_ok = Close(); assert(close_ok); } void -Fast_BufferedFile::ResetBuf(void) +Fast_BufferedFile::ResetBuf() { _bufi = buf(); _bufe = _bufi; } bool -Fast_BufferedFile::Close(void) +Fast_BufferedFile::Close() { Flush(); _openFlags = 0; @@ -426,14 +409,14 @@ Fast_BufferedFile::Close(void) bool Fast_BufferedFile::Open(unsigned int openFlags, const char * name) { - bool ok = false; + bool ok; if (openFlags & FASTOS_FILE_OPEN_READ) { - Close(); + ok = Close(); _filepos = 0; _fileleft = 0; ResetBuf(); - ok = _file->Open(openFlags, name); + ok &= _file->Open(openFlags, name); if (ok) { _openFlags = openFlags; //CASTWARN @@ -442,10 +425,10 @@ bool Fast_BufferedFile::Open(unsigned int openFlags, const char * name) // caller will have to check return value } } else { - Close(); + ok = Close(); _filepos = 0; ResetBuf(); - ok = _file->Open(FASTOS_FILE_OPEN_WRITE | openFlags, name); + ok &= _file->Open(FASTOS_FILE_OPEN_WRITE | openFlags, name); if (ok) { _openFlags = FASTOS_FILE_OPEN_WRITE | openFlags; } else { diff --git a/fastlib/src/vespa/fastlib/io/bufferedfile.h b/fastlib/src/vespa/fastlib/io/bufferedfile.h index 78c19ef8169..48f90262ad9 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedfile.h +++ b/fastlib/src/vespa/fastlib/io/bufferedfile.h @@ -26,7 +26,6 @@ private: /** True if the file should be read using direct IO */ bool _directIOEnabled; - void setupDirectIOAlign(); char * buf() { return static_cast(_buf.get()); } protected: /** The file instance used for low-level file access. */ @@ -51,7 +50,7 @@ public: /** * Delete the file instance used for low-level file access. **/ - virtual ~Fast_BufferedFile(void); + virtual ~Fast_BufferedFile(); /** * Open an existing file for reading. * @@ -75,13 +74,13 @@ public: * Reset the internal start and end pointers to the * head of the buffer, thus "emptying" it. */ - void ResetBuf(void); + void ResetBuf(); /** * Write the buffer to the file. Caution: Uses obsolete * FastOS_FileInterface::WriteBuf. * Allocates a 32kB buffer if not previously allocated. */ - void flushWriteBuf(void); + void flushWriteBuf(); /** * Read from the file into the buffer. Allocates a 32kB * buffer if not previously allocated. Fills the buffer, @@ -90,7 +89,7 @@ public: * Caution: If the amount read is smaller than the expected * amount, the method will abort. */ - void fillReadBuf(void); + void fillReadBuf(); /** * Read the next line of the buffered file into a buffer, * reading from the file as necessary. @@ -131,13 +130,7 @@ public: * @param byte The byte to write. */ void WriteByte(char byte); - /** - * Get one byte from the buffered file, reading from - * the file if necessary. - * - * @return int The byte read, or -1 if not read. - */ - int GetByte(void); + /** * Add an unsigned int number as ASCII text in base 10 to the buffered * file using a fixed width with a designated fill character. @@ -148,22 +141,14 @@ public: * for instance '0' or ' '. */ void addNum(unsigned int num, int fieldw, char fill); - /** - * Get the number of bytes left to read from the buffered - * file. This is the sum of bytes left in the buffer, and - * the number of bytes left in the file that has not yet - * been read into the buffer. - * - * @return The number of bytes left. - */ - uint64_t BytesLeft(void) const; + /** * Test for end of file. * * @return bool True if all bytes have been read from the * buffered file. */ - bool Eof(void) const; + bool Eof() const; /** * Get the size of the file. * diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index bd84a6ca419..a26e751ae73 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -5,16 +5,13 @@ #include #include #include -#include #include #include #include #include #include #include -#include #include -#include #include #include #include @@ -25,6 +22,7 @@ #include #include #include +#include #include #include @@ -41,6 +39,8 @@ using search::attribute::IAttributeVector; using vespalib::stringref; using vespalib::string; +namespace search { + namespace { string empty; @@ -157,8 +157,6 @@ bool contains_value(const Container& c, size_t elems, const V& value) { } -namespace search { - using attribute::CollectionType; using attribute::Config; diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 90d08fa681c..acd00413568 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -26,9 +26,6 @@ #include #include -class Fast_BufferedFile; -class FastOS_FileInterface; - namespace document { class ArithmeticValueUpdate; class AssignValueUpdate; diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp index 87f1985d7c0..f398fe0b46b 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp @@ -4,7 +4,6 @@ #include "attrvector.h" #include "load_utils.h" #include -#include #include namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp index 10f837ec1ab..73d248a21fa 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp @@ -8,7 +8,6 @@ #include "multinumericattributesaver.h" #include "load_utils.h" #include "primitivereader.h" -#include #include #include #include diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp index 5449f85bf68..4322faefe67 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp @@ -6,7 +6,6 @@ #include "load_utils.h" #include "loadednumericvalue.h" #include "multinumericenumattribute.h" -#include #include #include #include diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index 89cd8ca310a..c8330225fb9 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -6,7 +6,6 @@ #include "multistringattribute.h" #include "enumattribute.hpp" #include "multienumattribute.hpp" -#include #include #include #include diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index 2bb4d2ada60..a1f8b42df2f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -5,7 +5,6 @@ #include "stringattribute.h" #include "multistringpostattribute.h" #include "multistringattribute.hpp" -#include #include namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp index f08130b9fbe..d023d9b56b1 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp @@ -5,7 +5,6 @@ #include "readerbase.h" #include #include -#include #include #include diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp index 606c99161be..730ad1107a7 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp @@ -6,7 +6,6 @@ #include "stringattribute.h" #include "singleenumattribute.hpp" #include "attributevector.hpp" -#include #include #include #include diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp index d2b02b02a87..70309645ee2 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace search::diskindex { @@ -29,17 +30,13 @@ readHeader(vespalib::FileHeader &h, BitVectorFileWrite::BitVectorFileWrite(BitVectorKeyScope scope) : BitVectorIdxFileWrite(scope), - _datFile(nullptr), + _datFile(), _datHeaderLen(0) { } -BitVectorFileWrite::~BitVectorFileWrite() -{ - // No implicit close() call, but cleanup memory allocations. - delete _datFile; -} +BitVectorFileWrite::~BitVectorFileWrite() = default; void @@ -50,12 +47,11 @@ BitVectorFileWrite::open(const vespalib::string &name, { vespalib::string datname = name + ".bdat"; - assert(_datFile == nullptr); + assert( ! _datFile); Parent::open(name, docIdLimit, tuneFileWrite, fileHeaderContext); - FastOS_FileInterface *datfile = new FastOS_File; - _datFile = new Fast_BufferedFile(datfile); + _datFile = std::make_unique(new FastOS_File); if (tuneFileWrite.getWantSyncWrites()) { _datFile->EnableSyncWrites(); } @@ -115,13 +111,13 @@ BitVectorFileWrite::updateDatHeader(uint64_t fileBitSize) h.putTag(Tag("numKeys", _numKeys)); h.putTag(Tag("frozen", 1)); h.putTag(Tag("fileBitSize", fileBitSize)); - _datFile->Flush(); - _datFile->Sync(); + bool sync_ok = _datFile->Sync(); + assert(sync_ok); assert(h.getSize() == _datHeaderLen); _datFile->SetPosition(0); h.writeFile(*_datFile); - _datFile->Flush(); - _datFile->Sync(); + sync_ok = _datFile->Sync(); + assert(sync_ok); } @@ -150,7 +146,8 @@ BitVectorFileWrite::sync() { flush(); Parent::syncCommon(); - _datFile->Sync(); + bool sync_ok = _datFile->Sync(); + assert(sync_ok); } @@ -167,10 +164,10 @@ BitVectorFileWrite::close() (void) bitmapbytes; _datFile->alignEndForDirectIO(); updateDatHeader(pos * 8); - _datFile->Close(); + bool close_ok = _datFile->Close(); + assert(close_ok); } - delete _datFile; - _datFile = nullptr; + _datFile.reset(); } Parent::close(); } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h index 37ca72d3ec0..007368babdd 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h @@ -2,12 +2,13 @@ #pragma once -#include +#include "bitvectoridxfile.h" #include #include #include #include -#include "bitvectoridxfile.h" + +class Fast_BufferedFile; namespace search::diskindex { @@ -16,7 +17,7 @@ class BitVectorFileWrite : public BitVectorIdxFileWrite private: using Parent = BitVectorIdxFileWrite; - Fast_BufferedFile *_datFile; + std::unique_ptr _datFile; public: private: @@ -32,13 +33,13 @@ public: void open(const vespalib::string &name, uint32_t docIdLimit, const TuneFileSeqWrite &tuneFileWrite, - const common::FileHeaderContext &fileHeaderContext); + const common::FileHeaderContext &fileHeaderContext) override; void addWordSingle(uint64_t wordNum, const BitVector &bitVector); - void flush(); - void sync(); - void close(); + void flush() override; + void sync() override; + void close() override; void makeDatHeader(const common::FileHeaderContext &fileHeaderContext); void updateDatHeader(uint64_t fileBitSize); }; diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp index 17ad4c5b846..e87238bef2d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bitvectoridxfile.h" -#include #include #include #include #include #include +#include #include namespace search::diskindex { @@ -115,13 +115,13 @@ BitVectorIdxFileWrite::updateIdxHeader(uint64_t fileBitSize) if (_scope != BitVectorKeyScope::SHARED_WORDS) { h.putTag(Tag("fileBitSize", fileBitSize)); } - _idxFile->Flush(); - _idxFile->Sync(); + bool sync_ok = _idxFile->Sync(); + assert(sync_ok); assert(h.getSize() == _idxHeaderLen); _idxFile->SetPosition(0); h.writeFile(*_idxFile); - _idxFile->Flush(); - _idxFile->Sync(); + sync_ok = _idxFile->Sync(); + assert(sync_ok); } void @@ -147,7 +147,8 @@ BitVectorIdxFileWrite::flush() void BitVectorIdxFileWrite::syncCommon() { - _idxFile->Sync(); + bool sync_ok = _idxFile->Sync(); + assert(sync_ok); } void @@ -166,7 +167,8 @@ BitVectorIdxFileWrite::close() assert(pos == idxSize()); _idxFile->alignEndForDirectIO(); updateIdxHeader(pos * 8); - _idxFile->Close(); + bool close_ok = _idxFile->Close(); + assert(close_ok); } _idxFile.reset(); } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h index a5b6226fd43..f814cd20f5a 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h @@ -2,12 +2,13 @@ #pragma once -#include #include #include #include #include "bitvectorkeyscope.h" +class Fast_BufferedFile; + namespace search::common { class FileHeaderContext; } namespace search::diskindex { @@ -35,18 +36,18 @@ public: BitVectorIdxFileWrite& operator=(const BitVectorIdxFileWrite &&) = delete; BitVectorIdxFileWrite(BitVectorKeyScope scope); - ~BitVectorIdxFileWrite(); + virtual ~BitVectorIdxFileWrite(); - void open(const vespalib::string &name, uint32_t docIdLimit, - const TuneFileSeqWrite &tuneFileWrite, - const common::FileHeaderContext &fileHeaderContext); + virtual void open(const vespalib::string &name, uint32_t docIdLimit, + const TuneFileSeqWrite &tuneFileWrite, + const common::FileHeaderContext &fileHeaderContext); void addWordSingle(uint64_t wordNum, uint32_t numDocs); - void flush(); - void sync(); - void close(); + virtual void flush(); + virtual void sync(); + virtual void close(); static uint32_t getBitVectorLimit(uint32_t docIdLimit) { // Must match FastS_BinSizeParams::CalcMaxBinSize() diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp index a3b9259a278..7bc1bdbfb9b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp @@ -3,6 +3,7 @@ #include "dictionarywordreader.h" #include #include +#include #include LOG_SETUP(".diskindex.dictionarywordreader"); @@ -43,21 +44,27 @@ DictionaryWordReader::open(const vespalib::string & dictionaryName, _old2newwordfile->EnableDirectIO(); } // no checking possible - _old2newwordfile->WriteOpen(wordMapName.c_str()); + _old2newwordfile->OpenWriteOnly(wordMapName.c_str()); _old2newwordfile->SetSize(0); return true; } +void +DictionaryWordReader::writeNewWordNum(uint64_t newWordNum) { + _old2newwordfile->WriteBuf(&newWordNum, sizeof(newWordNum)); +} + void DictionaryWordReader::close() { if (!_dictFile->close()) { LOG(error, "Error closing input dictionary"); } - _old2newwordfile->Flush(); - _old2newwordfile->Sync(); - _old2newwordfile->Close(); + bool sync_ok = _old2newwordfile->Sync(); + assert(sync_ok); + bool close_ok = _old2newwordfile->Close(); + assert(close_ok); } } diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h index 18e886ca22e..5c5dc60f4e7 100644 --- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h @@ -2,7 +2,7 @@ #pragma once #include "pagedict4file.h" -#include + namespace search::diskindex { @@ -47,7 +47,7 @@ public: private: // "owners" of file handles. - std::unique_ptr _old2newwordfile; + std::unique_ptr _old2newwordfile; using DictionaryFileSeqRead = index::DictionaryFileSeqRead; std::unique_ptr _dictFile; @@ -82,9 +82,7 @@ public: void close(); - void writeNewWordNum(uint64_t newWordNum) { - _old2newwordfile->WriteBuf(&newWordNum, sizeof(newWordNum)); - } + void writeNewWordNum(uint64_t newWordNum); void write(WordAggregator &writer) { writer.tryWriteWord(_word); diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.h b/searchlib/src/vespa/searchlib/diskindex/zcposting.h index dc23fe5b37e..3d7dee1a988 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.h @@ -42,8 +42,6 @@ public: bool close() override; void getParams(PostingListParams ¶ms) override; void getFeatureParams(PostingListParams ¶ms) override; - void readWordStartWithSkip(); - void readWordStart(); void readHeader(); static const vespalib::string &getIdentifier(bool dynamic_k); }; diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 9a35e4a2b05..ae34cdd66c8 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(searchlib_tensor OBJECT SOURCES angular_distance.cpp bitvector_visited_tracker.cpp + blob_sequence_reader.cpp default_nearest_neighbor_index_factory.cpp dense_tensor_attribute.cpp dense_tensor_attribute_saver.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.cpp b/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.cpp new file mode 100644 index 00000000000..0d86af2f3a5 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.cpp @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "blob_sequence_reader.h" +#include + +namespace search::tensor { + +void +BlobSequenceReader::readBlob(void *buf, size_t len) { + _datFile.file().ReadBuf(buf, len); +} + +} // namespace diff --git a/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.h b/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.h index 5b7efc73a02..45fcf5524d2 100644 --- a/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.h +++ b/searchlib/src/vespa/searchlib/tensor/blob_sequence_reader.h @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include +#pragma once + #include -#include namespace search::tensor { @@ -20,7 +20,7 @@ public: _sizeReader(_datFile.file()) { } uint32_t getNextSize() { return _sizeReader.readHostOrder(); } - void readBlob(void *buf, size_t len) { _datFile.file().ReadBuf(buf, len); } + void readBlob(void *buf, size_t len); }; } // namespace diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp index 0cb23fe2ae9..0ae55a670da 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp @@ -5,7 +5,6 @@ #include #include -#include #include #include #include diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp index 16d9f2e3bbd..a0ec04b98cb 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp @@ -3,10 +3,7 @@ #include "serialized_fast_value_attribute.h" #include "streamed_value_saver.h" #include -#include #include -#include -#include #include #include diff --git a/searchlib/src/vespa/searchlib/util/fileutil.cpp b/searchlib/src/vespa/searchlib/util/fileutil.cpp index 83f3fe3a5fa..e85e792f492 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.cpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.cpp @@ -4,6 +4,7 @@ #include "filesizecalculator.h" #include #include +#include #include #include #include diff --git a/searchlib/src/vespa/searchlib/util/fileutil.h b/searchlib/src/vespa/searchlib/util/fileutil.h index 74096d8b5dd..8271265aa33 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.h +++ b/searchlib/src/vespa/searchlib/util/fileutil.h @@ -8,7 +8,6 @@ #include using vespalib::GenericHeader; -class Fast_BufferedFile; namespace search { diff --git a/searchlib/src/vespa/searchlib/util/fileutil.hpp b/searchlib/src/vespa/searchlib/util/fileutil.hpp index 98e54581ac7..5b5303ef169 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.hpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.hpp @@ -2,7 +2,6 @@ #pragma once #include "fileutil.h" -#include namespace search { -- cgit v1.2.3