diff options
author | Tor Egge <Tor.Egge@oath.com> | 2017-10-09 17:55:45 +0000 |
---|---|---|
committer | Tor Egge <Tor.Egge@oath.com> | 2017-10-09 17:55:45 +0000 |
commit | d318abe5c4880c235847dc8f72e5fadb245dbc8f (patch) | |
tree | c1afe135197615ab9e371012922e340eebb11cdd /searchlib | |
parent | 00cd7409531fb3e55d81edb92b94c29ff496a3de (diff) |
Remove support for disk index checkpointing.
Diffstat (limited to 'searchlib')
28 files changed, 40 insertions, 1126 deletions
diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index 08f1b8a1e9e..b56f4760ec7 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "compression.h" -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/data/fileheader.h> @@ -9,8 +8,6 @@ namespace search::bitcompression { -using vespalib::nbostream; - uint8_t CodingTables::_log2Table[65536]; CodingTables tables; // Static initializer @@ -129,33 +126,6 @@ EncodeContext64EBase<false>::writeBits(uint64_t data, uint32_t length) } } -void -EncodeContext64Base::checkPointWrite(nbostream &out) -{ - out << _cacheInt << _cacheFree; -} - - -void -EncodeContext64Base::checkPointRead(nbostream &in) -{ - in >> _cacheInt >> _cacheFree; -} - - -void -DecodeContext64Base::checkPointWrite(nbostream &out) -{ - (void) out; -} - - -void -DecodeContext64Base::checkPointRead(nbostream &in) -{ - (void) in; -} - namespace { vespalib::string noFeatures = "NoFeatures"; diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h index a0cba703e65..954afc60a1a 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h @@ -9,7 +9,6 @@ namespace vespalib { -class nbostream; class GenericHeader; } @@ -830,9 +829,6 @@ public: other._cacheFree = _cacheFree; } - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; - uint64_t getWriteOffset() const { return _fileWriteBias + (reinterpret_cast<unsigned long>(_valI) << 3) - _cacheFree; } @@ -1269,9 +1265,6 @@ public: return (_preRead == 0) ? 0 : 64 - _preRead; } - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; - static int64_t convertToSigned(uint64_t val) { if ((val & 1) != 0) return - (val >> 1) - 1; diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp index fca6c749cfb..49d5ae92ec8 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp @@ -3,14 +3,11 @@ #include "compression.h" #include "countcompression.h" #include <vespa/searchlib/index/postinglistcounts.h> -#include <vespa/vespalib/objects/nbostream.h> namespace search { namespace bitcompression { -using vespalib::nbostream; - #define K_VALUE_COUNTFILE_LASTDOCID 22 #define K_VALUE_COUNTFILE_NUMCHUNKS 1 #define K_VALUE_COUNTFILE_CHUNKNUMDOCS 18 @@ -19,24 +16,6 @@ using vespalib::nbostream; void -PostingListCountFileDecodeContext::checkPointWrite(nbostream &out) -{ - ParentClass::checkPointWrite(out); - out << _avgBitsPerDoc << _minChunkDocs << _docIdLimit << _numWordIds; - out << _minWordNum; -} - - -void -PostingListCountFileDecodeContext::checkPointRead(nbostream &in) -{ - ParentClass::checkPointRead(in); - in >> _avgBitsPerDoc >> _minChunkDocs >> _docIdLimit >> _numWordIds; - in >> _minWordNum; -} - - -void PostingListCountFileDecodeContext:: readCounts(PostingListCounts &counts) { @@ -145,24 +124,6 @@ copyParams(const PostingListCountFileDecodeContext &rhs) void -PostingListCountFileEncodeContext::checkPointWrite(nbostream &out) -{ - ParentClass::checkPointWrite(out); - out << _avgBitsPerDoc << _minChunkDocs << _docIdLimit << _numWordIds; - out << _minWordNum; -} - - -void -PostingListCountFileEncodeContext::checkPointRead(nbostream &in) -{ - ParentClass::checkPointRead(in); - in >> _avgBitsPerDoc >> _minChunkDocs >> _docIdLimit >> _numWordIds; - in >> _minWordNum; -} - - -void PostingListCountFileEncodeContext:: writeCounts(const PostingListCounts &counts) { diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.h b/searchlib/src/vespa/searchlib/bitcompression/countcompression.h index 4a514e58e2a..fc448ba1c30 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.h @@ -33,8 +33,6 @@ public: { } - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; void readCounts(PostingListCounts &counts); void readWordNum(uint64_t &wordNum); @@ -67,8 +65,6 @@ public: { } - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; void writeCounts(const PostingListCounts &counts); void writeWordNum(uint64_t wordNum); diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp index b3cba84d575..82110d354d3 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp @@ -5,7 +5,6 @@ #include "countcompression.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/dictionaryfile.h> -#include <vespa/vespalib/objects/nbostream.h> #include <sstream> #include <vespa/log/log.h> @@ -71,60 +70,6 @@ operator<<(std::ostream &stream, const index::PostingListCounts &counts) return stream; } -vespalib::nbostream & -operator<<(vespalib::nbostream &stream, - const PageDict4StartOffset &startOffset) -{ - stream << startOffset._fileOffset << startOffset._accNumDocs; - return stream; -} - -vespalib::nbostream & -operator>>(vespalib::nbostream &stream, PageDict4StartOffset &startOffset) -{ - stream >> startOffset._fileOffset >> startOffset._accNumDocs; - return stream; -} - - -vespalib::nbostream & -operator<<(vespalib::nbostream &stream, - const PageDict4SSReader::L7Entry &l7Entry) -{ - stream << l7Entry._l7Word << l7Entry._l7StartOffset << l7Entry._l7WordNum; - stream << l7Entry._l6Offset << l7Entry._sparsePageNum << l7Entry._pageNum; - stream << l7Entry._l7Ref; - return stream; -} - - -vespalib::nbostream & -operator>>(vespalib::nbostream &stream, - PageDict4SSReader::L7Entry &l7Entry) -{ - stream >> l7Entry._l7Word >> l7Entry._l7StartOffset >> l7Entry._l7WordNum; - stream >> l7Entry._l6Offset >> l7Entry._sparsePageNum >> l7Entry._pageNum; - stream >> l7Entry._l7Ref; - return stream; -} - - -vespalib::nbostream & -operator<<(vespalib::nbostream &stream, - const PageDict4SSReader::OverflowRef &oref) -{ - stream << oref._wordNum << oref._l7Ref; - return stream; -} - - -vespalib::nbostream & -operator>>(vespalib::nbostream &stream, PageDict4SSReader::OverflowRef &oref) -{ - stream >> oref._wordNum >> oref._l7Ref; - return stream; -} - typedef index::PostingListCounts Counts; typedef PageDict4StartOffset StartOffset; @@ -360,28 +305,6 @@ PageDict4SSWriter::flush() } -void -PageDict4SSWriter::checkPointWrite(vespalib::nbostream &out) -{ - out << _l6Word; - out << _l6StartOffset; - out << _l6PageNum; - out << _l6SparsePageNum; - out << _l6WordNum; -} - - -void -PageDict4SSWriter::checkPointRead(vespalib::nbostream &in) -{ - in >> _l6Word; - in >> _l6StartOffset; - in >> _l6PageNum; - in >> _l6SparsePageNum; - in >> _l6WordNum; -} - - PageDict4SPWriter::PageDict4SPWriter(SSWriter &ssWriter, EC &spe) : _eL3(), @@ -725,48 +648,6 @@ PageDict4SPWriter::addL5Skip(size_t &lcp) } -void -PageDict4SPWriter::checkPointWrite(vespalib::nbostream &out) -{ - _wcL3.checkPointWrite(out); - _wcL4.checkPointWrite(out); - _wcL5.checkPointWrite(out); - out << _l3Word << _l4Word << _l5Word << _l6Word; - out << _l3WordOffset << _l4WordOffset << _l5WordOffset; - out << _l3StartOffset << _l4StartOffset << _l5StartOffset << _l6StartOffset; - out << _l3WordNum << _l4WordNum << _l5WordNum << _l6WordNum; - out << _curL3OffsetL4 << _curL3OffsetL5 << _curL4OffsetL5; - out << _headerSize; - out << _l3Entries; - out << _l4StrideCheck << _l5StrideCheck; - out << _l3Size << _l4Size << _l5Size; - out << _prevL3Size << _prevL4Size << _prevL5Size << _prevWordsSize; - out << _sparsePageNum << _l3PageNum; - out << _words; -} - - -void -PageDict4SPWriter::checkPointRead(vespalib::nbostream &in) -{ - _wcL3.checkPointRead(in); - _wcL4.checkPointRead(in); - _wcL5.checkPointRead(in); - in >> _l3Word >> _l4Word >> _l5Word >> _l6Word; - in >> _l3WordOffset >> _l4WordOffset >> _l5WordOffset; - in >> _l3StartOffset >> _l4StartOffset >> _l5StartOffset >> _l6StartOffset; - in >> _l3WordNum >> _l4WordNum >> _l5WordNum >> _l6WordNum; - in >> _curL3OffsetL4 >> _curL3OffsetL5 >> _curL4OffsetL5; - in >> _headerSize; - in >> _l3Entries; - in >> _l4StrideCheck >> _l5StrideCheck; - in >> _l3Size >> _l4Size >> _l5Size; - in >> _prevL3Size >> _prevL4Size >> _prevL5Size >> _prevWordsSize; - in >> _sparsePageNum >> _l3PageNum; - in >> _words; -} - - PageDict4PWriter::PageDict4PWriter(SPWriter &spWriter, EC &pe) : _eCounts(), @@ -1148,52 +1029,6 @@ PageDict4PWriter::addL2Skip(size_t &lcp) } -void -PageDict4PWriter::checkPointWrite(vespalib::nbostream &out) -{ - _wcCounts.checkPointWrite(out); - _wcL1.checkPointWrite(out); - _wcL2.checkPointWrite(out); - out << _countsWord << _l1Word << _l2Word << _l3Word; - out << _pendingCountsWord; - out << _countsWordOffset << _l1WordOffset << _l2WordOffset; - out << _countsStartOffset << _l1StartOffset << _l2StartOffset; - out << _l3StartOffset; - out << _curCountOffsetL1 << _curCountOffsetL2 << _curL1OffsetL2; - out << _headerSize; - out << _countsEntries; - out << _l1StrideCheck << _l2StrideCheck; - out << _countsSize << _l1Size << _l2Size; - out << _prevL1Size << _prevL2Size; - out << _pageNum; - out << _l3WordNum << _wordNum; - out << _words; -} - - -void -PageDict4PWriter::checkPointRead(vespalib::nbostream &in) -{ - _wcCounts.checkPointRead(in); - _wcL1.checkPointRead(in); - _wcL2.checkPointRead(in); - in >> _countsWord >> _l1Word >> _l2Word >> _l3Word; - in >> _pendingCountsWord; - in >> _countsWordOffset >> _l1WordOffset >> _l2WordOffset; - in >> _countsStartOffset >> _l1StartOffset >> _l2StartOffset; - in >> _l3StartOffset; - in >> _curCountOffsetL1 >> _curCountOffsetL2 >> _curL1OffsetL2; - in >> _headerSize; - in >> _countsEntries; - in >> _l1StrideCheck >> _l2StrideCheck; - in >> _countsSize >> _l1Size >> _l2Size; - in >> _prevL1Size >> _prevL2Size; - in >> _pageNum; - in >> _l3WordNum >> _wordNum; - in >> _words; -} - - PageDict4SSLookupRes:: PageDict4SSLookupRes() : _l6Word(), @@ -1660,34 +1495,6 @@ lookupOverflow(uint64_t wordNum) const } -void -PageDict4SSReader::checkPointWrite(vespalib::nbostream &out) -{ - out << _ssFileBitLen << _ssStartOffset; - out << _l7; - _ssd.checkPointWrite(out); - out << _spFileBitLen << _pFileBitLen; - out << _spStartOffset << _pStartOffset; - out << _spFirstPageNum << _spFirstPageOffset; - out << _pFirstPageNum << _pFirstPageOffset; - out << _overflows; -} - - -void -PageDict4SSReader::checkPointRead(vespalib::nbostream &in) -{ - in >> _ssFileBitLen >> _ssStartOffset; - in >> _l7; - _ssd.checkPointRead(in); - in >> _spFileBitLen >> _pFileBitLen; - in >> _spStartOffset >> _pStartOffset; - in >> _spFirstPageNum >> _spFirstPageOffset; - in >> _pFirstPageNum >> _pFirstPageOffset; - in >> _overflows; -} - - PageDict4SPLookupRes:: PageDict4SPLookupRes() : _l3Word(), @@ -2512,68 +2319,6 @@ PageDict4Reader::readOverflowCounts(vespalib::string &word, _startOffset.adjust(counts); } -void -PageDict4Reader::checkPointWrite(vespalib::nbostream &out) -{ - out << _countsResidue; - out << _overflowPage; - out << _counts; - size_t ccOff = _cc - _counts.begin(); - size_t ceOff = _ce - _counts.begin(); - assert(ceOff == _counts.size()); - out << ccOff << ceOff; - out << _words; - size_t wcOff = _wc - _words.begin(); - size_t weOff = _we - _words.begin(); - assert(weOff = _words.size()); - out << wcOff << weOff; - out << _lastWord; - out << _lastSSWord; - out << _l3Residue; - out << _spwords; - size_t spwcOff = _spwc - _spwords.begin(); - size_t spweOff = _spwe - _spwords.begin(); - assert(spweOff == _spwords.size()); - out << spwcOff << spweOff; - _ssd.checkPointWrite(out); - out << _ssd.getReadOffset(); - out << _wordNum; -} - -void -PageDict4Reader::checkPointRead(vespalib::nbostream &in) -{ - in >> _countsResidue; - in >> _overflowPage; - in >> _counts; - size_t ccOff; - size_t ceOff; - in >> ccOff >> ceOff; - _cc = _counts.begin() + ccOff; - _ce = _counts.begin() + ceOff; - in >> _words; - size_t wcOff; - size_t weOff; - in >> wcOff >> weOff; - _wc = _words.begin() + wcOff; - _we = _words.begin() + weOff; - in >> _lastWord; - in >> _lastSSWord; - in >> _l3Residue; - in >> _spwords; - size_t spwcOff; - size_t spweOff; - in >> spwcOff >> spweOff; - _spwc = _spwords.begin() + spwcOff; - _spwe = _spwords.begin() + spweOff; - _ssd.checkPointRead(in); - int64_t ssReadOffset; - in >> ssReadOffset; - const ComprBuffer &sscb = _ssReader._cb; - setDecoderPosition(_ssd, sscb, ssReadOffset); - in >> _wordNum; -} - } // namespace bitcompression } // namespace search diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h index 2a4cd143cfa..47f2354bcc6 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h +++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h @@ -160,13 +160,6 @@ public: void flush(); - - - void - checkPointWrite(vespalib::nbostream &out); - - void - checkPointRead(vespalib::nbostream &in); }; @@ -307,12 +300,6 @@ public: { _ssWriter.addOverflowCounts(word, counts, startOffset, wordNum); } - - void - checkPointWrite(vespalib::nbostream &out); - - void - checkPointRead(vespalib::nbostream &in); }; /* @@ -447,12 +434,6 @@ public: { return _wordNum - 1; } - - void - checkPointWrite(vespalib::nbostream &out); - - void - checkPointRead(vespalib::nbostream &in); }; @@ -534,12 +515,6 @@ public: { return _l7Word < word; } - - friend vespalib::nbostream & - operator<<(vespalib::nbostream &stream, const L7Entry &l7Entry); - - friend vespalib::nbostream & - operator>>(vespalib::nbostream &stream, L7Entry &l7Entry); }; class OverflowRef @@ -565,12 +540,6 @@ public: { return _wordNum < wordNum; } - - friend vespalib::nbostream & - operator<<(vespalib::nbostream &stream, const OverflowRef &oref); - - friend vespalib::nbostream & - operator>>(vespalib::nbostream &stream, OverflowRef &oref); }; ComprBuffer _cb; @@ -617,12 +586,6 @@ public: { return _ssd; } - - void - checkPointWrite(vespalib::nbostream &out); - - void - checkPointRead(vespalib::nbostream &in); }; @@ -752,12 +715,6 @@ public: void readOverflowCounts(vespalib::string &word, Counts &counts); - - void - checkPointWrite(vespalib::nbostream &out); - - void - checkPointRead(vespalib::nbostream &in); }; } // namespace bitcompression diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp index defacdb05d7..374c6a8c7e6 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp @@ -4,12 +4,10 @@ #include <vespa/searchlib/index/bitvectorkeys.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/fileheadercontext.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/fileheader.h> namespace search::diskindex { -using vespalib::nbostream; using search::index::BitVectorWordSingleKey; using search::common::FileHeaderContext; @@ -45,24 +43,6 @@ BitVectorFileWrite::~BitVectorFileWrite() void -BitVectorFileWrite::checkPointWrite(nbostream &out) -{ - flush(); - Parent::checkPointWriteCommon(out); - out << _datHeaderLen; - sync(); -} - - -void -BitVectorFileWrite::checkPointRead(nbostream &in) -{ - Parent::checkPointRead(in); - in >> _datHeaderLen; -} - - -void BitVectorFileWrite::open(const vespalib::string &name, uint32_t docIdLimit, const TuneFileSeqWrite &tuneFileWrite, @@ -198,33 +178,4 @@ BitVectorFileWrite::close() BitVectorCandidate::~BitVectorCandidate() { } -void -BitVectorCandidate::checkPointWrite(nbostream &out) -{ - uint32_t docIdLimit = _bv->size(); - out << docIdLimit << _numDocs << _bitVectorLimit; - out.saveVector(_array); - if (getCrossedBitVectorLimit()) - out << *_bv; -} - - -void -BitVectorCandidate::checkPointRead(nbostream &in) -{ - uint32_t docIdLimit = _bv->size(); - uint32_t checkDocIdLimit; - uint32_t checkBitVectorLimit; - in >> checkDocIdLimit >> _numDocs >> checkBitVectorLimit; - assert(checkDocIdLimit == docIdLimit); - (void) docIdLimit; - assert(checkBitVectorLimit == _bitVectorLimit); - in.restoreVector(_array); - if (getCrossedBitVectorLimit()) { - in >> *_bv; - } else { - _bv->clear(); - } -} - } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h index 1b7b522bc52..893b792d96b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h @@ -37,21 +37,6 @@ public: ~BitVectorFileWrite(); - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - void - checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void - checkPointRead(vespalib::nbostream &in); - void open(const vespalib::string &name, uint32_t docIdLimit, const TuneFileSeqWrite &tuneFileWrite, @@ -181,21 +166,6 @@ public: { return *_bv; } - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - void - checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void - checkPointRead(vespalib::nbostream &in); }; diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp index 856c527a8b5..a4eca7feac9 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp @@ -4,12 +4,10 @@ #include <vespa/searchlib/index/bitvectorkeys.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/fileheadercontext.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/fileheader.h> namespace search::diskindex { -using vespalib::nbostream; using search::index::BitVectorWordSingleKey; using search::common::FileHeaderContext; @@ -55,35 +53,6 @@ BitVectorIdxFileWrite::idxSize() const void -BitVectorIdxFileWrite::checkPointWriteCommon(nbostream &out) -{ - out << _scope; - out << _docIdLimit << _numKeys; - out << _idxHeaderLen; -} - - -void -BitVectorIdxFileWrite::checkPointWrite(nbostream &out) -{ - flush(); - checkPointWriteCommon(out); - sync(); -} - - -void -BitVectorIdxFileWrite::checkPointRead(nbostream &in) -{ - BitVectorKeyScope checkScope; - in >> checkScope; - assert(checkScope == _scope); - in >> _docIdLimit >> _numKeys; - in >> _idxHeaderLen; -} - - -void BitVectorIdxFileWrite::open(const vespalib::string &name, uint32_t docIdLimit, const TuneFileSeqWrite &tuneFileWrite, diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h index 0ab40d9c4a3..8d704d1bffe 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h @@ -8,13 +8,6 @@ #include <vespa/vespalib/stllike/string.h> #include "bitvectorkeyscope.h" -namespace vespalib -{ - -class nbostream; - -} - namespace search { @@ -49,7 +42,6 @@ protected: BitVectorKeyScope _scope; uint64_t idxSize() const; - void checkPointWriteCommon(vespalib::nbostream &out); void syncCommon(); public: @@ -57,21 +49,6 @@ public: ~BitVectorIdxFileWrite(); - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - void - checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void - checkPointRead(vespalib::nbostream &in); - void open(const vespalib::string &name, uint32_t docIdLimit, const TuneFileSeqWrite &tuneFileWrite, diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp index c1964906f85..108defea6a0 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp @@ -5,7 +5,6 @@ #include "extposocc.h" #include "pagedict4file.h" #include <vespa/vespalib/util/error.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".diskindex.fieldreader"); @@ -36,7 +35,6 @@ FieldReader::FieldReader() _oldWordNum(noWordNumHigh()), _residue(0u), _docIdLimit(0u), - _checkPointResume(false), _word() { } @@ -154,13 +152,11 @@ FieldReader::lateOpen(const vespalib::string &prefix, "Could not open posocc file %s for read", name.c_str()); } - if (!_checkPointResume) { - _oldWordNum = noWordNum(); - _wordNum = _oldWordNum; - PostingListParams params; - _oldposoccfile->getParams(params); - params.get("docIdLimit", _docIdLimit); - } + _oldWordNum = noWordNum(); + _wordNum = _oldWordNum; + PostingListParams params; + _oldposoccfile->getParams(params); + params.get("docIdLimit", _docIdLimit); return true; } @@ -204,29 +200,6 @@ FieldReader::close() void -FieldReader::checkPointWrite(vespalib::nbostream &out) -{ - out << _wordNum << _oldWordNum; - out << _residue << _docIdAndFeatures; - out << _docIdLimit; - out << _word; - _oldposoccfile->checkPointWrite(out); - _dictFile->checkPointWrite(out); -} - -void -FieldReader::checkPointRead(vespalib::nbostream &in) -{ - in >> _wordNum >> _oldWordNum; - in >> _residue >> _docIdAndFeatures; - in >> _docIdLimit; - in >> _word; - _oldposoccfile->checkPointRead(in); - _dictFile->checkPointRead(in); - _checkPointResume = true; -} - -void FieldReader::setFeatureParams(const PostingListParams ¶ms) { _oldposoccfile->setFeatureParams(params); diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h index 3d724f31b41..aebb2fbe53a 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h @@ -56,7 +56,6 @@ protected: uint64_t _oldWordNum; uint32_t _residue; uint32_t _docIdLimit; - bool _checkPointResume; vespalib::string _word; static uint64_t @@ -130,18 +129,6 @@ public: virtual bool close(); - /* - * To be called between words, not in the middle of one. - */ - virtual void - checkPointWrite(vespalib::nbostream &out); - - /* - * To be called after earlyOpen() but before afterOpen(). - */ - virtual void - checkPointRead(vespalib::nbostream &in); - virtual void setFeatureParams(const PostingListParams ¶ms); diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp index 9f03c06cf56..8b0c60d0bd7 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp @@ -5,14 +5,12 @@ #include "extposocc.h" #include "pagedict4file.h" #include <vespa/vespalib/util/error.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".diskindex.fieldwriter"); namespace search::diskindex { -using vespalib::nbostream; using vespalib::getLastErrorString; using common::FileHeaderContext; @@ -181,36 +179,6 @@ FieldWriter::close() void -FieldWriter::checkPointWrite(nbostream &out) -{ - out << _wordNum << _prevDocId; - out << _docIdLimit << _numWordIds; - out << _compactWordNum << _word; - _posoccfile->checkPointWrite(out); - _dictFile->checkPointWrite(out); - _bvc.checkPointWrite(out); - _bmapfile.checkPointWrite(out); -} - - -void -FieldWriter::checkPointRead(nbostream &in) -{ - in >> _wordNum >> _prevDocId; - uint32_t checkDocIdLimit = 0; - uint64_t checkNumWordIds = 0; - in >> checkDocIdLimit >> checkNumWordIds; - assert(checkDocIdLimit == _docIdLimit); - assert(checkNumWordIds == _numWordIds); - in >> _compactWordNum >> _word; - _posoccfile->checkPointRead(in); - _dictFile->checkPointRead(in); - _bvc.checkPointRead(in); - _bmapfile.checkPointRead(in); -} - - -void FieldWriter::setFeatureParams(const PostingListParams ¶ms) { _posoccfile->setFeatureParams(params); diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h index 1cd53b99bce..893572fbdad 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h @@ -8,8 +8,6 @@ #include <vespa/searchlib/bitcompression/countcompression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> -namespace vespalib { class nbostream; } - namespace search { namespace diskindex { @@ -82,15 +80,6 @@ public: bool close(); - /* - * To be called between words, not in the middle of one. - */ - void checkPointWrite(vespalib::nbostream &out); - - /* - * To be called after earlyOpen() but before afterOpen(). - */ - void checkPointRead(vespalib::nbostream &in); void setFeatureParams(const PostingListParams ¶ms); void getFeatureParams(PostingListParams ¶ms); static void remove(const vespalib::string &prefix); diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index 97d21caef96..8d6b9d04b51 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -321,7 +321,6 @@ Fusion::openFieldWriter(const SchemaUtil::IndexIterator &index, index.getSchema(), index.getIndex(), _tuneFileIndexing._write); - // No checkpointing if (!writer.lateOpen(_tuneFileIndexing._write, _fileHeaderContext)) { LOG(error, "Could not open output posocc + dictionary in %s", diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp index aacfa400755..0fdc4b96202 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp @@ -296,8 +296,6 @@ FileHandle::open(const vespalib::stringref &dir, index.getSchema(), index.getIndex(), tuneFileWrite); - // No checkpointing - if (!_fieldWriter->lateOpen(tuneFileWrite, fileHeaderContext)) { LOG(error, "Could not open term writer %s for write (%s)", dir.c_str(), getLastErrorString().c_str()); diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp index 9c63c5930b6..72fd07e5752 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp @@ -4,7 +4,6 @@ #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/log/log.h> @@ -63,8 +62,7 @@ PageDict4FileSeqRead::PageDict4FileSeqRead() _ssCompleted(false), _spCompleted(false), _pCompleted(false), - _wordNum(0u), - _checkPointData() + _wordNum(0u) { _ssd.setReadContext(&_ssReadContext); _spd.setReadContext(&_spReadContext); @@ -229,26 +227,13 @@ PageDict4FileSeqRead::open(const vespalib::string &name, 32768u); _ssd.emptyBuffer(0); - if (_checkPointData) { - _ssReadContext.setPosition(_ssReadContext.getCheckPointOffset()); - if (_ssd._valI >= _ssd._valE) - _ssReadContext.readComprBuffer(); - _spReadContext.setPosition(_spReadContext.getCheckPointOffset()); - if (_spd._valI >= _spd._valE) - _spReadContext.readComprBuffer(); - _pReadContext.setPosition(_pReadContext.getCheckPointOffset()); - if (_pd._valI >= _pd._valE) - _pReadContext.readComprBuffer(); - } else { - _ssReadContext.readComprBuffer(); - assert(_ssReadContext.getBufferEndFilePos() >= fileSize); - readSSHeader(); - _spReadContext.readComprBuffer(); - readSPHeader(); - _pReadContext.readComprBuffer(); - readPHeader(); - } - + _ssReadContext.readComprBuffer(); + assert(_ssReadContext.getBufferEndFilePos() >= fileSize); + readSSHeader(); + _spReadContext.readComprBuffer(); + readSPHeader(); + _pReadContext.readComprBuffer(); + readPHeader(); _ssReader = new SSReader(_ssReadContext, _ssHeaderLen, @@ -263,15 +248,9 @@ PageDict4FileSeqRead::open(const vespalib::string &name, _spd, _pd); - if (_checkPointData) { - _ssReader->checkPointRead(*_checkPointData); - _pReader->checkPointRead(*_checkPointData); - assert(_checkPointData->empty()); - } else { - _ssReader->setup(_ssd); - _pReader->setup(); - _wordNum = 0; - } + _ssReader->setup(_ssd); + _pReader->setup(); + _wordNum = 0; return true; } @@ -299,40 +278,6 @@ PageDict4FileSeqRead::close() void -PageDict4FileSeqRead::checkPointWrite(vespalib::nbostream &out) -{ - _ssd.checkPointWrite(out); - _spReadContext.checkPointWrite(out); - _pReadContext.checkPointWrite(out); - vespalib::nbostream data; - _ssReader->checkPointWrite(data); - _pReader->checkPointWrite(data); - std::vector<char> checkPointData(data.size()); - data.read(&checkPointData[0], data.size()); - out << checkPointData; - out << _wordNum; - out << _ssCompleted << _ssFileBitSize << _ssHeaderLen; - out << _spCompleted << _spFileBitSize << _spHeaderLen; - out << _pCompleted << _pFileBitSize << _pHeaderLen; -} - -void -PageDict4FileSeqRead::checkPointRead(vespalib::nbostream &in) -{ - _ssd.checkPointRead(in); - _spReadContext.checkPointRead(in); - _pReadContext.checkPointRead(in); - std::vector<char> checkPointData; - in >> checkPointData; - _checkPointData.reset(new vespalib::nbostream(checkPointData.size())); - _checkPointData->write(&checkPointData[0], checkPointData.size()); - in >> _wordNum; - in >> _ssCompleted >> _ssFileBitSize >> _ssHeaderLen; - in >> _spCompleted >> _spFileBitSize >> _spHeaderLen; - in >> _pCompleted >> _pFileBitSize >> _pHeaderLen; -} - -void PageDict4FileSeqRead::getParams(PostingListParams ¶ms) { params.clear(); @@ -418,11 +363,9 @@ PageDict4FileSeqWrite::open(const vespalib::string &name, assertOpenWriteOnly(ok, ssname); _ssWriteContext.setFile(&_ssfile); - if (!_checkPointData) { - _pWriteContext.allocComprBuf(65536u, 32768u); - _spWriteContext.allocComprBuf(65536u, 32768u); - _ssWriteContext.allocComprBuf(65536u, 32768u); - } + _pWriteContext.allocComprBuf(65536u, 32768u); + _spWriteContext.allocComprBuf(65536u, 32768u); + _ssWriteContext.allocComprBuf(65536u, 32768u); uint64_t pFileSize = _pfile.GetSize(); uint64_t spFileSize = _spfile.GetSize(); @@ -445,33 +388,24 @@ PageDict4FileSeqWrite::open(const vespalib::string &name, assert(ssBufferStartFilePos == static_cast<uint64_t>(_ssfile.GetPosition())); - if (!_checkPointData) { - _pe.setupWrite(_pWriteContext); - _spe.setupWrite(_spWriteContext); - _sse.setupWrite(_ssWriteContext); - assert(_pe.getWriteOffset() == 0); - assert(_spe.getWriteOffset() == 0); - assert(_sse.getWriteOffset() == 0); - _spe.copyParams(_sse); - _pe.copyParams(_sse); - // Write initial file headers - makePHeader(fileHeaderContext); - makeSPHeader(fileHeaderContext); - makeSSHeader(fileHeaderContext); - } + _pe.setupWrite(_pWriteContext); + _spe.setupWrite(_spWriteContext); + _sse.setupWrite(_ssWriteContext); + assert(_pe.getWriteOffset() == 0); + assert(_spe.getWriteOffset() == 0); + assert(_sse.getWriteOffset() == 0); + _spe.copyParams(_sse); + _pe.copyParams(_sse); + // Write initial file headers + makePHeader(fileHeaderContext); + makeSPHeader(fileHeaderContext); + makeSSHeader(fileHeaderContext); _ssWriter = new SSWriter(_sse); _spWriter = new SPWriter(*_ssWriter, _spe); _pWriter = new PWriter(*_spWriter, _pe); - if (_checkPointData) { - _ssWriter->checkPointRead(*_checkPointData); - _spWriter->checkPointRead(*_checkPointData); - _pWriter->checkPointRead(*_checkPointData); - assert(_checkPointData->empty()); - } else { - _spWriter->setup(); - _pWriter->setup(); - } + _spWriter->setup(); + _pWriter->setup(); return true; } @@ -683,43 +617,6 @@ PageDict4FileSeqWrite::updateSSHeader(uint64_t fileBitSize) void -PageDict4FileSeqWrite::checkPointWrite(vespalib::nbostream &out) -{ - _ssWriteContext.writeComprBuffer(true); - _spWriteContext.writeComprBuffer(true); - _pWriteContext.writeComprBuffer(true); - _ssWriteContext.checkPointWrite(out); - _spWriteContext.checkPointWrite(out); - _pWriteContext.checkPointWrite(out); - vespalib::nbostream data; - _ssWriter->checkPointWrite(data); - _spWriter->checkPointWrite(data); - _pWriter->checkPointWrite(data); - std::vector<char> checkPointData(data.size()); - data.read(&checkPointData[0], data.size()); - out << checkPointData; - out << _ssHeaderLen << _spHeaderLen << _pHeaderLen; - _ssfile.Sync(); - _spfile.Sync(); - _pfile.Sync(); -} - - -void -PageDict4FileSeqWrite::checkPointRead(vespalib::nbostream &in) -{ - _ssWriteContext.checkPointRead(in); - _spWriteContext.checkPointRead(in); - _pWriteContext.checkPointRead(in); - std::vector<char> checkPointData; - in >> checkPointData; - _checkPointData.reset(new vespalib::nbostream(checkPointData.size())); - _checkPointData->write(&checkPointData[0], checkPointData.size()); - in >> _ssHeaderLen >> _spHeaderLen >> _pHeaderLen; -} - - -void PageDict4FileSeqWrite::setParams(const PostingListParams ¶ms) { params.get("avgBitsPerDoc", _sse._avgBitsPerDoc); diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h index fe432a22463..7baa883a740 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h @@ -50,8 +50,6 @@ class PageDict4FileSeqRead : public index::DictionaryFileSeqRead uint64_t _wordNum; - std::unique_ptr<vespalib::nbostream> _checkPointData; - void readSSHeader(); @@ -85,21 +83,6 @@ public: */ virtual bool close() override; - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void - checkPointWrite(vespalib::nbostream &out) override; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void - checkPointRead(vespalib::nbostream &in) override; - /* * Get current parameters. */ @@ -142,8 +125,6 @@ class PageDict4FileSeqWrite : public index::DictionaryFileSeqWrite uint32_t _spHeaderLen; // Length of header for sparse page file (bytes) uint32_t _ssHeaderLen; // Length of header for sparse sparse file (bytes) - std::unique_ptr<vespalib::nbostream> _checkPointData; - void writeIndexNames(vespalib::GenericHeader &header); @@ -196,21 +177,6 @@ public: virtual bool close() override; - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void - checkPointWrite(vespalib::nbostream &out) override; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void - checkPointRead(vespalib::nbostream &in) override; - /* * Set parameters. */ diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index 768ee972097..17d4457a318 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -6,7 +6,6 @@ #include <vespa/searchlib/index/postinglistfile.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/common/fileheadercontext.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/log/log.h> @@ -27,7 +26,6 @@ using index::PostingListCountFileSeqWrite; using common::FileHeaderContext; using bitcompression::FeatureDecodeContextBE; using bitcompression::FeatureEncodeContextBE; -using vespalib::nbostream; using vespalib::getLastErrorString; @@ -78,11 +76,7 @@ Zc4PostingSeqRead(PostingListCountFileSeqRead *countFile) _rangeEndOffset(0), _readAheadEndOffset(0), _wordStart(0), - _checkPointPos(0), - _residue(0), - _checkPointChunkNo(0u), - _checkPointResidue(0u), - _checkPointHasMore(false) + _residue(0) { if (_countFile != NULL) { PostingListParams params; @@ -231,32 +225,6 @@ readDocIdAndFeatures(DocIdAndFeatures &features) void -Zc4PostingSeqRead::checkPointWrite(nbostream &out) -{ - out << _counts; - out << _wordStart; - uint64_t curPos = _decodeContext->getReadOffset(); - out << curPos; - out << _residue; - out << _chunkNo; - out << _hasMore; -} - - -void -Zc4PostingSeqRead::checkPointRead(nbostream &in) -{ - in >> _counts; - in >> _wordStart; - in >> _checkPointPos; - in >> _checkPointResidue; - in >> _checkPointChunkNo; - in >> _checkPointHasMore; - assert(_checkPointPos >= _wordStart); -} - - -void Zc4PostingSeqRead::readWordStartWithSkip() { typedef FeatureEncodeContextBE EC; @@ -508,34 +476,8 @@ Zc4PostingSeqRead::open(const vespalib::string &name, _readContext.readComprBuffer(); readHeader(); - if (d._valI >= d._valE) + if (d._valI >= d._valE) { _readContext.readComprBuffer(); - if (_checkPointPos != 0) { - if (_checkPointResidue != 0 || _checkPointHasMore) { - // Checkpointed in the middle of a word. Read from - // start at word until at right position. - DocIdAndFeatures features; - _readContext.setPosition(_wordStart); - assert(_decodeContext->getReadOffset() == _wordStart); - _readContext.readComprBuffer(); - readWordStart(); - assert(_chunkNo < _checkPointChunkNo || - (_chunkNo == _checkPointChunkNo && - _residue >= _checkPointResidue)); - while (_chunkNo < _checkPointChunkNo || - _residue > _checkPointResidue) { - readDocIdAndFeatures(features); - } - assert(_chunkNo == _checkPointChunkNo); - assert(_residue == _checkPointResidue); - assert(_hasMore == _checkPointHasMore); - assert(_decodeContext->getReadOffset() == _checkPointPos); - } else { - // Checkpointed between words. - _readContext.setPosition(_checkPointPos); - assert(_decodeContext->getReadOffset() == _checkPointPos); - _readContext.readComprBuffer(); - } } } else { LOG(error, "could not open %s: %s", @@ -740,29 +682,6 @@ Zc4PostingSeqWrite::flushWord() } -void -Zc4PostingSeqWrite::checkPointWrite(nbostream &out) -{ - _writeContext.writeComprBuffer(true); // Also flush slack - out << _numWords; - _writeContext.checkPointWrite(out); - _featureWriteContext.checkPointWrite(out); - out.saveVector(_docIds) << _writePos << _counts; - _file.Sync(); -} - - -void -Zc4PostingSeqWrite::checkPointRead(nbostream &in) -{ - in >> _numWords; - _writeContext.checkPointRead(in); - _featureWriteContext.checkPointRead(in); - _featureOffset = _encodeFeatures->getWriteOffset(); - in.restoreVector(_docIds) >> _writePos >> _counts; -} - - uint32_t Zc4PostingSeqWrite::readHeader(const vespalib::string &name) { @@ -1422,10 +1341,3 @@ ZcPostingSeqWrite::flushWordNoSkip() } } // namespace search::diskindex - -#include <vespa/vespalib/objects/nbostream.hpp> -namespace vespalib { - using UInt32Pair = std::pair<uint32_t, uint32_t>; - template nbostream &nbostream::saveVector<UInt32Pair>(const std::vector<UInt32Pair> &); - template nbostream &nbostream::restoreVector<UInt32Pair>(std::vector<UInt32Pair> &); -} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.h b/searchlib/src/vespa/searchlib/diskindex/zcposting.h index 64879aa05fb..9a650abdd40 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.h @@ -78,11 +78,7 @@ protected: uint64_t _rangeEndOffset; // End offset for word pair uint64_t _readAheadEndOffset; // Readahead end offset for word pair uint64_t _wordStart; // last word header position - uint64_t _checkPointPos; // file position when checkpointing uint32_t _residue; // Number of unread documents after word header - uint32_t _checkPointChunkNo; // _chunkNo when checkpointing - uint32_t _checkPointResidue; // _residue when checkpointing - bool _checkPointHasMore; // _hasMore when checkpointing public: Zc4PostingSeqRead(index::PostingListCountFileSeqRead *countFile); @@ -98,8 +94,6 @@ public: virtual void readCommonWordDocIdAndFeatures(DocIdAndFeatures &features); void readDocIdAndFeatures(DocIdAndFeatures &features) override; - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; void readCounts(const PostingListCounts &counts) override; // Fill in for next word bool open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) override; bool close() override; @@ -176,8 +170,6 @@ public: void writeDocIdAndFeatures(const DocIdAndFeatures &features) override; void flushWord() override; - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; bool open(const vespalib::string &name, const TuneFileSeqWrite &tuneFileWrite, const search::common::FileHeaderContext &fileHeaderContext) override; diff --git a/searchlib/src/vespa/searchlib/index/postinglistcountfile.h b/searchlib/src/vespa/searchlib/index/postinglistcountfile.h index 5b14b477c42..30c6963f228 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistcountfile.h +++ b/searchlib/src/vespa/searchlib/index/postinglistcountfile.h @@ -5,8 +5,6 @@ #include "postinglistcounts.h" #include <vespa/searchlib/common/tunefileinfo.h> -namespace vespalib { class nbostream; } - namespace search { namespace common { class FileHeaderContext; } @@ -34,19 +32,6 @@ public: virtual ~PostingListCountFileSeqRead(); /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; - - /** * Open posting list count file for sequential read. */ virtual bool open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) = 0; @@ -71,19 +56,6 @@ public: virtual ~PostingListCountFileSeqWrite(); /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; - - /** * Open posting list count file for sequential write. */ virtual bool open(const vespalib::string &name, diff --git a/searchlib/src/vespa/searchlib/index/postinglistfile.h b/searchlib/src/vespa/searchlib/index/postinglistfile.h index 43628bd798b..61247ef3197 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistfile.h +++ b/searchlib/src/vespa/searchlib/index/postinglistfile.h @@ -8,8 +8,6 @@ class FastOS_FileInterface; -namespace vespalib { class nbostream; } - namespace search { namespace common { class FileHeaderContext; } @@ -38,19 +36,6 @@ public: virtual void readDocIdAndFeatures(DocIdAndFeatures &features) = 0; /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; - - /** * Read counts for a word. */ virtual void readCounts(const PostingListCounts &counts) = 0; @@ -135,19 +120,6 @@ public: virtual void flushWord() = 0; /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; - - /** * Open posting list file for sequential write. */ virtual bool diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp index 8c7f97db022..acfc2e46020 100644 --- a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp +++ b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp @@ -1,14 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "comprbuffer.h" -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/fastos/file.h> #include <cassert> +#include <cstring> namespace search { -using vespalib::nbostream; - ComprBuffer::ComprBuffer(uint32_t unitSize) : _comprBuf(NULL), _comprBufSize(0), @@ -114,25 +112,4 @@ ComprBuffer::referenceComprBuf(const ComprBuffer &rhs) _comprBufSize = rhs._comprBufSize; } - -void -ComprBuffer::checkPointWrite(nbostream &out) -{ - _aligner.checkPointWrite(out); - out << _comprBufSize << _unitSize << _padBefore; -} - - -void -ComprBuffer::checkPointRead(nbostream &in) -{ - _aligner.checkPointRead(in); - uint32_t unitSize; - in >> _comprBufSize >> unitSize >> _padBefore; - assert(unitSize == _unitSize); - - allocComprBuf(); -} - - } diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.h b/searchlib/src/vespa/searchlib/util/comprbuffer.h index 61928976acc..c15671d5e14 100644 --- a/searchlib/src/vespa/searchlib/util/comprbuffer.h +++ b/searchlib/src/vespa/searchlib/util/comprbuffer.h @@ -31,7 +31,6 @@ public: static size_t minimumPadding() { return 8; } uint32_t getUnitBitSize() const { return _unitSize * 8; } bool getPadBefore() const { return _padBefore; } - bool getCheckPointResumed() const { return _aligner.getCheckPointResumed(); } /* * When encoding to memory instead of file, the compressed buffer must @@ -44,18 +43,6 @@ public: * long as rhs is live and unchanged. */ void referenceComprBuf(const ComprBuffer &rhs); - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. - */ - void checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - * - */ - void checkPointRead(vespalib::nbostream &in); }; } diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp index 8b17fd85753..26c66f43993 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.cpp +++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp @@ -1,14 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "comprfile.h" -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/fastos/file.h> #include <cassert> +#include <cstring> namespace search { -using vespalib::nbostream; - void ComprFileReadBase::ReadComprBuffer(uint64_t stopOffset, bool readAll, @@ -292,9 +290,7 @@ ComprFileReadContext(ComprFileDecodeContext &decodeContext) _bitOffset(0), _stopOffset(0), _readAll(true), - _checkPointOffsetValid(false), - _file(NULL), - _checkPointOffset(0) + _file(NULL) { } @@ -308,9 +304,7 @@ ComprFileReadContext(uint32_t unitSize) _bitOffset(0), _stopOffset(0), _readAll(true), - _checkPointOffsetValid(false), - _file(NULL), - _checkPointOffset(0) + _file(NULL) { } @@ -497,28 +491,6 @@ ComprFileReadContext::copyReadContext(const ComprFileReadContext &rhs) } } - -void -ComprFileReadContext::checkPointWrite(nbostream &out) -{ - ComprBuffer::checkPointWrite(out); - ComprFileDecodeContext &d = *_decodeContext; - d.checkPointWrite(out); - uint64_t bitOffset = d.getBitPosV(); - out << bitOffset; -} - - -void -ComprFileReadContext::checkPointRead(nbostream &in) -{ - ComprBuffer::checkPointRead(in); - ComprFileDecodeContext &d = *_decodeContext; - d.checkPointRead(in); - in >> _checkPointOffset; // Cannot seek until file is opened - _checkPointOffsetValid = true; -} - ComprFileWriteContext:: ComprFileWriteContext(ComprFileEncodeContext &encodeContext) : ComprBuffer(encodeContext.getUnitByteSize()), @@ -598,47 +570,4 @@ ComprFileWriteContext::allocComprBuf() allocComprBuf(32768, 32768); } - -void -ComprFileWriteContext::checkPointWrite(nbostream &out) -{ - ComprBuffer::checkPointWrite(out); - ComprFileEncodeContext &e = *_encodeContext; - uint64_t bufferStartFilePos = getBufferStartFilePos(); - uint64_t usedSize = e.getUsedUnits(_comprBuf) * - e.getUnitByteSize(); - out << bufferStartFilePos << usedSize; - e.checkPointWrite(out); - if (usedSize != 0) { - out.write(_comprBuf, usedSize); - } - uint64_t bitOffset = e.getBitPosV(); - out << bitOffset; -} - - -void -ComprFileWriteContext::checkPointRead(nbostream &in) -{ - ComprBuffer::checkPointRead(in); - ComprFileEncodeContext &e = *_encodeContext; - uint64_t bufferStartFilePos = 0; - uint64_t usedSize = 0; - in >> bufferStartFilePos >> usedSize; - e.checkPointRead(in); - if (usedSize != 0) { - assert((usedSize % e.getUnitByteSize()) == 0); - assert(_comprBufSize >= usedSize / e.getUnitByteSize()); - in.read(_comprBuf, usedSize); - } - setBufferStartFilePos(bufferStartFilePos); - e.afterWrite(*this, usedSize / e.getUnitByteSize(), bufferStartFilePos); - uint64_t bitOffset = 0; - in >> bitOffset; - uint64_t writeOffset = e.getBitPosV(); - assert(bitOffset == writeOffset); - (void) writeOffset; -} - - } diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h index b69ef140868..3d44f088c74 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.h +++ b/searchlib/src/vespa/searchlib/util/comprfile.h @@ -7,8 +7,6 @@ class FastOS_FileInterface; -namespace vespalib { class nbostream; } - namespace search { class ComprFileWriteContext; @@ -60,19 +58,6 @@ public: * Get size of each unit (typically 4 or 8) */ virtual uint32_t getUnitByteSize() const = 0; - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Caller must - * save position. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - * Caller must restore position. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; }; class ComprFileReadBase @@ -110,9 +95,7 @@ private: int _bitOffset; uint64_t _stopOffset; bool _readAll; - bool _checkPointOffsetValid; // Set only if checkpoint has been read FastOS_FileInterface *_file; - uint64_t _checkPointOffset; // bit offset saved by checkPointRead public: ComprFileReadContext(ComprFileDecodeContext &decodeContext); @@ -170,19 +153,6 @@ public: * For unit testing only. Copy data owned by rhs. */ void copyWriteContext(const ComprFileWriteContext &rhs); - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. - */ - void checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void checkPointRead(vespalib::nbostream &in); - bool getCheckPointOffsetValid() const { return _checkPointOffsetValid; } - uint64_t getCheckPointOffset() const { return _checkPointOffset; } }; @@ -216,19 +186,6 @@ public: */ virtual uint32_t getUnitByteSize() const = 0; - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Caller must - * save position, although partial unit is saved. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - * Caller must restore positon, although partial unit is restored. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; - virtual uint64_t getBitPosV() const = 0; }; @@ -281,17 +238,6 @@ public: * no file is attached. */ std::pair<void *, size_t> grabComprBuffer(void *&comprBufMalloc); - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. - */ - void checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void checkPointRead(vespalib::nbostream &in); }; } diff --git a/searchlib/src/vespa/searchlib/util/filealign.cpp b/searchlib/src/vespa/searchlib/util/filealign.cpp index 15bef611714..b60c0492854 100644 --- a/searchlib/src/vespa/searchlib/util/filealign.cpp +++ b/searchlib/src/vespa/searchlib/util/filealign.cpp @@ -1,14 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "filealign.h" -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/fastos/file.h> #include <cassert> namespace search { -using vespalib::nbostream; - namespace { size_t @@ -42,8 +39,7 @@ FileAlign::FileAlign() _minAlignedSize(1), _elemSize(1), _directIOMemAlign(1), - _directio(false), - _checkPointResumed(false) + _directio(false) { } @@ -116,24 +112,4 @@ FileAlign::setupAlign(size_t elements, return elements; } - -void -FileAlign::checkPointWrite(nbostream &out) -{ - out << _directIOFileAlign << _preferredFileAlign << - _minDirectIOSize << _minAlignedSize << _elemSize << - _directIOMemAlign << _directio; -} - - -void -FileAlign::checkPointRead(nbostream &in) -{ - in >> _directIOFileAlign >> _preferredFileAlign >> - _minDirectIOSize >> _minAlignedSize >> _elemSize >> - _directIOMemAlign >> _directio; - _checkPointResumed = true; -} - - } diff --git a/searchlib/src/vespa/searchlib/util/filealign.h b/searchlib/src/vespa/searchlib/util/filealign.h index 3e75fb2f371..49d053957ce 100644 --- a/searchlib/src/vespa/searchlib/util/filealign.h +++ b/searchlib/src/vespa/searchlib/util/filealign.h @@ -7,8 +7,6 @@ class FastOS_FileInterface; -namespace vespalib { class nbostream; } - namespace search { class FileAlign @@ -21,7 +19,6 @@ private: size_t _elemSize; size_t _directIOMemAlign; bool _directio; - bool _checkPointResumed; public: @@ -64,24 +61,12 @@ public: */ size_t setupAlign(size_t elements, size_t elemSize, FastOS_FileInterface *file, size_t preferredFileAlignment); bool getDirectIO() const { return _directio; } - bool getCheckPointResumed() const { return _checkPointResumed; } size_t getDirectIOFileAlign() const { return _directIOFileAlign; } size_t getDirectIOMemAlign() const { return _directIOMemAlign; } size_t getMinDirectIOSize() const { return _minDirectIOSize; } size_t getMinAlignedSize() const { return _minAlignedSize; } size_t getPreferredFileAlign() const { return _preferredFileAlign; } size_t getElemSize() const { return _elemSize; } - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. - */ - void checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void checkPointRead(vespalib::nbostream &in); }; } |