summaryrefslogtreecommitdiffstats
path: root/searchlib
diff options
context:
space:
mode:
authorTor Egge <Tor.Egge@oath.com>2017-10-09 17:55:45 +0000
committerTor Egge <Tor.Egge@oath.com>2017-10-09 17:55:45 +0000
commitd318abe5c4880c235847dc8f72e5fadb245dbc8f (patch)
treec1afe135197615ab9e371012922e340eebb11cdd /searchlib
parent00cd7409531fb3e55d81edb92b94c29ff496a3de (diff)
Remove support for disk index checkpointing.
Diffstat (limited to 'searchlib')
-rw-r--r--searchlib/src/vespa/searchlib/bitcompression/compression.cpp30
-rw-r--r--searchlib/src/vespa/searchlib/bitcompression/compression.h7
-rw-r--r--searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp39
-rw-r--r--searchlib/src/vespa/searchlib/bitcompression/countcompression.h4
-rw-r--r--searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp255
-rw-r--r--searchlib/src/vespa/searchlib/bitcompression/pagedict4.h43
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp49
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h30
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp31
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h23
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp37
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/fieldreader.h13
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp32
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/fieldwriter.h11
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/fusion.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp159
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/pagedict4file.h34
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/zcposting.cpp92
-rw-r--r--searchlib/src/vespa/searchlib/diskindex/zcposting.h8
-rw-r--r--searchlib/src/vespa/searchlib/index/postinglistcountfile.h28
-rw-r--r--searchlib/src/vespa/searchlib/index/postinglistfile.h28
-rw-r--r--searchlib/src/vespa/searchlib/util/comprbuffer.cpp25
-rw-r--r--searchlib/src/vespa/searchlib/util/comprbuffer.h13
-rw-r--r--searchlib/src/vespa/searchlib/util/comprfile.cpp77
-rw-r--r--searchlib/src/vespa/searchlib/util/comprfile.h54
-rw-r--r--searchlib/src/vespa/searchlib/util/filealign.cpp26
-rw-r--r--searchlib/src/vespa/searchlib/util/filealign.h15
28 files changed, 40 insertions, 1126 deletions
diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp
index 08f1b8a1e9e..b56f4760ec7 100644
--- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp
+++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "compression.h"
-#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/searchlib/fef/termfieldmatchdata.h>
#include <vespa/searchlib/fef/termfieldmatchdataarray.h>
#include <vespa/vespalib/data/fileheader.h>
@@ -9,8 +8,6 @@
namespace search::bitcompression {
-using vespalib::nbostream;
-
uint8_t CodingTables::_log2Table[65536];
CodingTables tables; // Static initializer
@@ -129,33 +126,6 @@ EncodeContext64EBase<false>::writeBits(uint64_t data, uint32_t length)
}
}
-void
-EncodeContext64Base::checkPointWrite(nbostream &out)
-{
- out << _cacheInt << _cacheFree;
-}
-
-
-void
-EncodeContext64Base::checkPointRead(nbostream &in)
-{
- in >> _cacheInt >> _cacheFree;
-}
-
-
-void
-DecodeContext64Base::checkPointWrite(nbostream &out)
-{
- (void) out;
-}
-
-
-void
-DecodeContext64Base::checkPointRead(nbostream &in)
-{
- (void) in;
-}
-
namespace {
vespalib::string noFeatures = "NoFeatures";
diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h
index a0cba703e65..954afc60a1a 100644
--- a/searchlib/src/vespa/searchlib/bitcompression/compression.h
+++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h
@@ -9,7 +9,6 @@
namespace vespalib {
-class nbostream;
class GenericHeader;
}
@@ -830,9 +829,6 @@ public:
other._cacheFree = _cacheFree;
}
- void checkPointWrite(vespalib::nbostream &out) override;
- void checkPointRead(vespalib::nbostream &in) override;
-
uint64_t getWriteOffset() const {
return _fileWriteBias + (reinterpret_cast<unsigned long>(_valI) << 3) - _cacheFree;
}
@@ -1269,9 +1265,6 @@ public:
return (_preRead == 0) ? 0 : 64 - _preRead;
}
- void checkPointWrite(vespalib::nbostream &out) override;
- void checkPointRead(vespalib::nbostream &in) override;
-
static int64_t convertToSigned(uint64_t val) {
if ((val & 1) != 0)
return - (val >> 1) - 1;
diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp
index fca6c749cfb..49d5ae92ec8 100644
--- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp
+++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp
@@ -3,14 +3,11 @@
#include "compression.h"
#include "countcompression.h"
#include <vespa/searchlib/index/postinglistcounts.h>
-#include <vespa/vespalib/objects/nbostream.h>
namespace search {
namespace bitcompression {
-using vespalib::nbostream;
-
#define K_VALUE_COUNTFILE_LASTDOCID 22
#define K_VALUE_COUNTFILE_NUMCHUNKS 1
#define K_VALUE_COUNTFILE_CHUNKNUMDOCS 18
@@ -19,24 +16,6 @@ using vespalib::nbostream;
void
-PostingListCountFileDecodeContext::checkPointWrite(nbostream &out)
-{
- ParentClass::checkPointWrite(out);
- out << _avgBitsPerDoc << _minChunkDocs << _docIdLimit << _numWordIds;
- out << _minWordNum;
-}
-
-
-void
-PostingListCountFileDecodeContext::checkPointRead(nbostream &in)
-{
- ParentClass::checkPointRead(in);
- in >> _avgBitsPerDoc >> _minChunkDocs >> _docIdLimit >> _numWordIds;
- in >> _minWordNum;
-}
-
-
-void
PostingListCountFileDecodeContext::
readCounts(PostingListCounts &counts)
{
@@ -145,24 +124,6 @@ copyParams(const PostingListCountFileDecodeContext &rhs)
void
-PostingListCountFileEncodeContext::checkPointWrite(nbostream &out)
-{
- ParentClass::checkPointWrite(out);
- out << _avgBitsPerDoc << _minChunkDocs << _docIdLimit << _numWordIds;
- out << _minWordNum;
-}
-
-
-void
-PostingListCountFileEncodeContext::checkPointRead(nbostream &in)
-{
- ParentClass::checkPointRead(in);
- in >> _avgBitsPerDoc >> _minChunkDocs >> _docIdLimit >> _numWordIds;
- in >> _minWordNum;
-}
-
-
-void
PostingListCountFileEncodeContext::
writeCounts(const PostingListCounts &counts)
{
diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.h b/searchlib/src/vespa/searchlib/bitcompression/countcompression.h
index 4a514e58e2a..fc448ba1c30 100644
--- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.h
+++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.h
@@ -33,8 +33,6 @@ public:
{
}
- void checkPointWrite(vespalib::nbostream &out) override;
- void checkPointRead(vespalib::nbostream &in) override;
void readCounts(PostingListCounts &counts);
void readWordNum(uint64_t &wordNum);
@@ -67,8 +65,6 @@ public:
{
}
- void checkPointWrite(vespalib::nbostream &out) override;
- void checkPointRead(vespalib::nbostream &in) override;
void writeCounts(const PostingListCounts &counts);
void writeWordNum(uint64_t wordNum);
diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp
index b3cba84d575..82110d354d3 100644
--- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp
+++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp
@@ -5,7 +5,6 @@
#include "countcompression.h"
#include <vespa/searchlib/index/postinglistcounts.h>
#include <vespa/searchlib/index/dictionaryfile.h>
-#include <vespa/vespalib/objects/nbostream.h>
#include <sstream>
#include <vespa/log/log.h>
@@ -71,60 +70,6 @@ operator<<(std::ostream &stream, const index::PostingListCounts &counts)
return stream;
}
-vespalib::nbostream &
-operator<<(vespalib::nbostream &stream,
- const PageDict4StartOffset &startOffset)
-{
- stream << startOffset._fileOffset << startOffset._accNumDocs;
- return stream;
-}
-
-vespalib::nbostream &
-operator>>(vespalib::nbostream &stream, PageDict4StartOffset &startOffset)
-{
- stream >> startOffset._fileOffset >> startOffset._accNumDocs;
- return stream;
-}
-
-
-vespalib::nbostream &
-operator<<(vespalib::nbostream &stream,
- const PageDict4SSReader::L7Entry &l7Entry)
-{
- stream << l7Entry._l7Word << l7Entry._l7StartOffset << l7Entry._l7WordNum;
- stream << l7Entry._l6Offset << l7Entry._sparsePageNum << l7Entry._pageNum;
- stream << l7Entry._l7Ref;
- return stream;
-}
-
-
-vespalib::nbostream &
-operator>>(vespalib::nbostream &stream,
- PageDict4SSReader::L7Entry &l7Entry)
-{
- stream >> l7Entry._l7Word >> l7Entry._l7StartOffset >> l7Entry._l7WordNum;
- stream >> l7Entry._l6Offset >> l7Entry._sparsePageNum >> l7Entry._pageNum;
- stream >> l7Entry._l7Ref;
- return stream;
-}
-
-
-vespalib::nbostream &
-operator<<(vespalib::nbostream &stream,
- const PageDict4SSReader::OverflowRef &oref)
-{
- stream << oref._wordNum << oref._l7Ref;
- return stream;
-}
-
-
-vespalib::nbostream &
-operator>>(vespalib::nbostream &stream, PageDict4SSReader::OverflowRef &oref)
-{
- stream >> oref._wordNum >> oref._l7Ref;
- return stream;
-}
-
typedef index::PostingListCounts Counts;
typedef PageDict4StartOffset StartOffset;
@@ -360,28 +305,6 @@ PageDict4SSWriter::flush()
}
-void
-PageDict4SSWriter::checkPointWrite(vespalib::nbostream &out)
-{
- out << _l6Word;
- out << _l6StartOffset;
- out << _l6PageNum;
- out << _l6SparsePageNum;
- out << _l6WordNum;
-}
-
-
-void
-PageDict4SSWriter::checkPointRead(vespalib::nbostream &in)
-{
- in >> _l6Word;
- in >> _l6StartOffset;
- in >> _l6PageNum;
- in >> _l6SparsePageNum;
- in >> _l6WordNum;
-}
-
-
PageDict4SPWriter::PageDict4SPWriter(SSWriter &ssWriter,
EC &spe)
: _eL3(),
@@ -725,48 +648,6 @@ PageDict4SPWriter::addL5Skip(size_t &lcp)
}
-void
-PageDict4SPWriter::checkPointWrite(vespalib::nbostream &out)
-{
- _wcL3.checkPointWrite(out);
- _wcL4.checkPointWrite(out);
- _wcL5.checkPointWrite(out);
- out << _l3Word << _l4Word << _l5Word << _l6Word;
- out << _l3WordOffset << _l4WordOffset << _l5WordOffset;
- out << _l3StartOffset << _l4StartOffset << _l5StartOffset << _l6StartOffset;
- out << _l3WordNum << _l4WordNum << _l5WordNum << _l6WordNum;
- out << _curL3OffsetL4 << _curL3OffsetL5 << _curL4OffsetL5;
- out << _headerSize;
- out << _l3Entries;
- out << _l4StrideCheck << _l5StrideCheck;
- out << _l3Size << _l4Size << _l5Size;
- out << _prevL3Size << _prevL4Size << _prevL5Size << _prevWordsSize;
- out << _sparsePageNum << _l3PageNum;
- out << _words;
-}
-
-
-void
-PageDict4SPWriter::checkPointRead(vespalib::nbostream &in)
-{
- _wcL3.checkPointRead(in);
- _wcL4.checkPointRead(in);
- _wcL5.checkPointRead(in);
- in >> _l3Word >> _l4Word >> _l5Word >> _l6Word;
- in >> _l3WordOffset >> _l4WordOffset >> _l5WordOffset;
- in >> _l3StartOffset >> _l4StartOffset >> _l5StartOffset >> _l6StartOffset;
- in >> _l3WordNum >> _l4WordNum >> _l5WordNum >> _l6WordNum;
- in >> _curL3OffsetL4 >> _curL3OffsetL5 >> _curL4OffsetL5;
- in >> _headerSize;
- in >> _l3Entries;
- in >> _l4StrideCheck >> _l5StrideCheck;
- in >> _l3Size >> _l4Size >> _l5Size;
- in >> _prevL3Size >> _prevL4Size >> _prevL5Size >> _prevWordsSize;
- in >> _sparsePageNum >> _l3PageNum;
- in >> _words;
-}
-
-
PageDict4PWriter::PageDict4PWriter(SPWriter &spWriter,
EC &pe)
: _eCounts(),
@@ -1148,52 +1029,6 @@ PageDict4PWriter::addL2Skip(size_t &lcp)
}
-void
-PageDict4PWriter::checkPointWrite(vespalib::nbostream &out)
-{
- _wcCounts.checkPointWrite(out);
- _wcL1.checkPointWrite(out);
- _wcL2.checkPointWrite(out);
- out << _countsWord << _l1Word << _l2Word << _l3Word;
- out << _pendingCountsWord;
- out << _countsWordOffset << _l1WordOffset << _l2WordOffset;
- out << _countsStartOffset << _l1StartOffset << _l2StartOffset;
- out << _l3StartOffset;
- out << _curCountOffsetL1 << _curCountOffsetL2 << _curL1OffsetL2;
- out << _headerSize;
- out << _countsEntries;
- out << _l1StrideCheck << _l2StrideCheck;
- out << _countsSize << _l1Size << _l2Size;
- out << _prevL1Size << _prevL2Size;
- out << _pageNum;
- out << _l3WordNum << _wordNum;
- out << _words;
-}
-
-
-void
-PageDict4PWriter::checkPointRead(vespalib::nbostream &in)
-{
- _wcCounts.checkPointRead(in);
- _wcL1.checkPointRead(in);
- _wcL2.checkPointRead(in);
- in >> _countsWord >> _l1Word >> _l2Word >> _l3Word;
- in >> _pendingCountsWord;
- in >> _countsWordOffset >> _l1WordOffset >> _l2WordOffset;
- in >> _countsStartOffset >> _l1StartOffset >> _l2StartOffset;
- in >> _l3StartOffset;
- in >> _curCountOffsetL1 >> _curCountOffsetL2 >> _curL1OffsetL2;
- in >> _headerSize;
- in >> _countsEntries;
- in >> _l1StrideCheck >> _l2StrideCheck;
- in >> _countsSize >> _l1Size >> _l2Size;
- in >> _prevL1Size >> _prevL2Size;
- in >> _pageNum;
- in >> _l3WordNum >> _wordNum;
- in >> _words;
-}
-
-
PageDict4SSLookupRes::
PageDict4SSLookupRes()
: _l6Word(),
@@ -1660,34 +1495,6 @@ lookupOverflow(uint64_t wordNum) const
}
-void
-PageDict4SSReader::checkPointWrite(vespalib::nbostream &out)
-{
- out << _ssFileBitLen << _ssStartOffset;
- out << _l7;
- _ssd.checkPointWrite(out);
- out << _spFileBitLen << _pFileBitLen;
- out << _spStartOffset << _pStartOffset;
- out << _spFirstPageNum << _spFirstPageOffset;
- out << _pFirstPageNum << _pFirstPageOffset;
- out << _overflows;
-}
-
-
-void
-PageDict4SSReader::checkPointRead(vespalib::nbostream &in)
-{
- in >> _ssFileBitLen >> _ssStartOffset;
- in >> _l7;
- _ssd.checkPointRead(in);
- in >> _spFileBitLen >> _pFileBitLen;
- in >> _spStartOffset >> _pStartOffset;
- in >> _spFirstPageNum >> _spFirstPageOffset;
- in >> _pFirstPageNum >> _pFirstPageOffset;
- in >> _overflows;
-}
-
-
PageDict4SPLookupRes::
PageDict4SPLookupRes()
: _l3Word(),
@@ -2512,68 +2319,6 @@ PageDict4Reader::readOverflowCounts(vespalib::string &word,
_startOffset.adjust(counts);
}
-void
-PageDict4Reader::checkPointWrite(vespalib::nbostream &out)
-{
- out << _countsResidue;
- out << _overflowPage;
- out << _counts;
- size_t ccOff = _cc - _counts.begin();
- size_t ceOff = _ce - _counts.begin();
- assert(ceOff == _counts.size());
- out << ccOff << ceOff;
- out << _words;
- size_t wcOff = _wc - _words.begin();
- size_t weOff = _we - _words.begin();
- assert(weOff = _words.size());
- out << wcOff << weOff;
- out << _lastWord;
- out << _lastSSWord;
- out << _l3Residue;
- out << _spwords;
- size_t spwcOff = _spwc - _spwords.begin();
- size_t spweOff = _spwe - _spwords.begin();
- assert(spweOff == _spwords.size());
- out << spwcOff << spweOff;
- _ssd.checkPointWrite(out);
- out << _ssd.getReadOffset();
- out << _wordNum;
-}
-
-void
-PageDict4Reader::checkPointRead(vespalib::nbostream &in)
-{
- in >> _countsResidue;
- in >> _overflowPage;
- in >> _counts;
- size_t ccOff;
- size_t ceOff;
- in >> ccOff >> ceOff;
- _cc = _counts.begin() + ccOff;
- _ce = _counts.begin() + ceOff;
- in >> _words;
- size_t wcOff;
- size_t weOff;
- in >> wcOff >> weOff;
- _wc = _words.begin() + wcOff;
- _we = _words.begin() + weOff;
- in >> _lastWord;
- in >> _lastSSWord;
- in >> _l3Residue;
- in >> _spwords;
- size_t spwcOff;
- size_t spweOff;
- in >> spwcOff >> spweOff;
- _spwc = _spwords.begin() + spwcOff;
- _spwe = _spwords.begin() + spweOff;
- _ssd.checkPointRead(in);
- int64_t ssReadOffset;
- in >> ssReadOffset;
- const ComprBuffer &sscb = _ssReader._cb;
- setDecoderPosition(_ssd, sscb, ssReadOffset);
- in >> _wordNum;
-}
-
} // namespace bitcompression
} // namespace search
diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h
index 2a4cd143cfa..47f2354bcc6 100644
--- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h
+++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h
@@ -160,13 +160,6 @@ public:
void
flush();
-
-
- void
- checkPointWrite(vespalib::nbostream &out);
-
- void
- checkPointRead(vespalib::nbostream &in);
};
@@ -307,12 +300,6 @@ public:
{
_ssWriter.addOverflowCounts(word, counts, startOffset, wordNum);
}
-
- void
- checkPointWrite(vespalib::nbostream &out);
-
- void
- checkPointRead(vespalib::nbostream &in);
};
/*
@@ -447,12 +434,6 @@ public:
{
return _wordNum - 1;
}
-
- void
- checkPointWrite(vespalib::nbostream &out);
-
- void
- checkPointRead(vespalib::nbostream &in);
};
@@ -534,12 +515,6 @@ public:
{
return _l7Word < word;
}
-
- friend vespalib::nbostream &
- operator<<(vespalib::nbostream &stream, const L7Entry &l7Entry);
-
- friend vespalib::nbostream &
- operator>>(vespalib::nbostream &stream, L7Entry &l7Entry);
};
class OverflowRef
@@ -565,12 +540,6 @@ public:
{
return _wordNum < wordNum;
}
-
- friend vespalib::nbostream &
- operator<<(vespalib::nbostream &stream, const OverflowRef &oref);
-
- friend vespalib::nbostream &
- operator>>(vespalib::nbostream &stream, OverflowRef &oref);
};
ComprBuffer _cb;
@@ -617,12 +586,6 @@ public:
{
return _ssd;
}
-
- void
- checkPointWrite(vespalib::nbostream &out);
-
- void
- checkPointRead(vespalib::nbostream &in);
};
@@ -752,12 +715,6 @@ public:
void
readOverflowCounts(vespalib::string &word,
Counts &counts);
-
- void
- checkPointWrite(vespalib::nbostream &out);
-
- void
- checkPointRead(vespalib::nbostream &in);
};
} // namespace bitcompression
diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp
index defacdb05d7..374c6a8c7e6 100644
--- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp
@@ -4,12 +4,10 @@
#include <vespa/searchlib/index/bitvectorkeys.h>
#include <vespa/searchlib/common/bitvector.h>
#include <vespa/searchlib/common/fileheadercontext.h>
-#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/data/fileheader.h>
namespace search::diskindex {
-using vespalib::nbostream;
using search::index::BitVectorWordSingleKey;
using search::common::FileHeaderContext;
@@ -45,24 +43,6 @@ BitVectorFileWrite::~BitVectorFileWrite()
void
-BitVectorFileWrite::checkPointWrite(nbostream &out)
-{
- flush();
- Parent::checkPointWriteCommon(out);
- out << _datHeaderLen;
- sync();
-}
-
-
-void
-BitVectorFileWrite::checkPointRead(nbostream &in)
-{
- Parent::checkPointRead(in);
- in >> _datHeaderLen;
-}
-
-
-void
BitVectorFileWrite::open(const vespalib::string &name,
uint32_t docIdLimit,
const TuneFileSeqWrite &tuneFileWrite,
@@ -198,33 +178,4 @@ BitVectorFileWrite::close()
BitVectorCandidate::~BitVectorCandidate() {
}
-void
-BitVectorCandidate::checkPointWrite(nbostream &out)
-{
- uint32_t docIdLimit = _bv->size();
- out << docIdLimit << _numDocs << _bitVectorLimit;
- out.saveVector(_array);
- if (getCrossedBitVectorLimit())
- out << *_bv;
-}
-
-
-void
-BitVectorCandidate::checkPointRead(nbostream &in)
-{
- uint32_t docIdLimit = _bv->size();
- uint32_t checkDocIdLimit;
- uint32_t checkBitVectorLimit;
- in >> checkDocIdLimit >> _numDocs >> checkBitVectorLimit;
- assert(checkDocIdLimit == docIdLimit);
- (void) docIdLimit;
- assert(checkBitVectorLimit == _bitVectorLimit);
- in.restoreVector(_array);
- if (getCrossedBitVectorLimit()) {
- in >> *_bv;
- } else {
- _bv->clear();
- }
-}
-
}
diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h
index 1b7b522bc52..893b792d96b 100644
--- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h
+++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h
@@ -37,21 +37,6 @@ public:
~BitVectorFileWrite();
- /**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt. Implies
- * flush from memory to disk, and possibly also sync to permanent
- * storage media.
- */
- void
- checkPointWrite(vespalib::nbostream &out);
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- void
- checkPointRead(vespalib::nbostream &in);
-
void
open(const vespalib::string &name, uint32_t docIdLimit,
const TuneFileSeqWrite &tuneFileWrite,
@@ -181,21 +166,6 @@ public:
{
return *_bv;
}
-
- /**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt. Implies
- * flush from memory to disk, and possibly also sync to permanent
- * storage media.
- */
- void
- checkPointWrite(vespalib::nbostream &out);
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- void
- checkPointRead(vespalib::nbostream &in);
};
diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp
index 856c527a8b5..a4eca7feac9 100644
--- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp
@@ -4,12 +4,10 @@
#include <vespa/searchlib/index/bitvectorkeys.h>
#include <vespa/searchlib/common/bitvector.h>
#include <vespa/searchlib/common/fileheadercontext.h>
-#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/data/fileheader.h>
namespace search::diskindex {
-using vespalib::nbostream;
using search::index::BitVectorWordSingleKey;
using search::common::FileHeaderContext;
@@ -55,35 +53,6 @@ BitVectorIdxFileWrite::idxSize() const
void
-BitVectorIdxFileWrite::checkPointWriteCommon(nbostream &out)
-{
- out << _scope;
- out << _docIdLimit << _numKeys;
- out << _idxHeaderLen;
-}
-
-
-void
-BitVectorIdxFileWrite::checkPointWrite(nbostream &out)
-{
- flush();
- checkPointWriteCommon(out);
- sync();
-}
-
-
-void
-BitVectorIdxFileWrite::checkPointRead(nbostream &in)
-{
- BitVectorKeyScope checkScope;
- in >> checkScope;
- assert(checkScope == _scope);
- in >> _docIdLimit >> _numKeys;
- in >> _idxHeaderLen;
-}
-
-
-void
BitVectorIdxFileWrite::open(const vespalib::string &name,
uint32_t docIdLimit,
const TuneFileSeqWrite &tuneFileWrite,
diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h
index 0ab40d9c4a3..8d704d1bffe 100644
--- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h
+++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h
@@ -8,13 +8,6 @@
#include <vespa/vespalib/stllike/string.h>
#include "bitvectorkeyscope.h"
-namespace vespalib
-{
-
-class nbostream;
-
-}
-
namespace search
{
@@ -49,7 +42,6 @@ protected:
BitVectorKeyScope _scope;
uint64_t idxSize() const;
- void checkPointWriteCommon(vespalib::nbostream &out);
void syncCommon();
public:
@@ -57,21 +49,6 @@ public:
~BitVectorIdxFileWrite();
- /**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt. Implies
- * flush from memory to disk, and possibly also sync to permanent
- * storage media.
- */
- void
- checkPointWrite(vespalib::nbostream &out);
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- void
- checkPointRead(vespalib::nbostream &in);
-
void
open(const vespalib::string &name, uint32_t docIdLimit,
const TuneFileSeqWrite &tuneFileWrite,
diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp
index c1964906f85..108defea6a0 100644
--- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp
@@ -5,7 +5,6 @@
#include "extposocc.h"
#include "pagedict4file.h"
#include <vespa/vespalib/util/error.h>
-#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/log/log.h>
LOG_SETUP(".diskindex.fieldreader");
@@ -36,7 +35,6 @@ FieldReader::FieldReader()
_oldWordNum(noWordNumHigh()),
_residue(0u),
_docIdLimit(0u),
- _checkPointResume(false),
_word()
{
}
@@ -154,13 +152,11 @@ FieldReader::lateOpen(const vespalib::string &prefix,
"Could not open posocc file %s for read",
name.c_str());
}
- if (!_checkPointResume) {
- _oldWordNum = noWordNum();
- _wordNum = _oldWordNum;
- PostingListParams params;
- _oldposoccfile->getParams(params);
- params.get("docIdLimit", _docIdLimit);
- }
+ _oldWordNum = noWordNum();
+ _wordNum = _oldWordNum;
+ PostingListParams params;
+ _oldposoccfile->getParams(params);
+ params.get("docIdLimit", _docIdLimit);
return true;
}
@@ -204,29 +200,6 @@ FieldReader::close()
void
-FieldReader::checkPointWrite(vespalib::nbostream &out)
-{
- out << _wordNum << _oldWordNum;
- out << _residue << _docIdAndFeatures;
- out << _docIdLimit;
- out << _word;
- _oldposoccfile->checkPointWrite(out);
- _dictFile->checkPointWrite(out);
-}
-
-void
-FieldReader::checkPointRead(vespalib::nbostream &in)
-{
- in >> _wordNum >> _oldWordNum;
- in >> _residue >> _docIdAndFeatures;
- in >> _docIdLimit;
- in >> _word;
- _oldposoccfile->checkPointRead(in);
- _dictFile->checkPointRead(in);
- _checkPointResume = true;
-}
-
-void
FieldReader::setFeatureParams(const PostingListParams &params)
{
_oldposoccfile->setFeatureParams(params);
diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h
index 3d724f31b41..aebb2fbe53a 100644
--- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h
+++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h
@@ -56,7 +56,6 @@ protected:
uint64_t _oldWordNum;
uint32_t _residue;
uint32_t _docIdLimit;
- bool _checkPointResume;
vespalib::string _word;
static uint64_t
@@ -130,18 +129,6 @@ public:
virtual bool
close();
- /*
- * To be called between words, not in the middle of one.
- */
- virtual void
- checkPointWrite(vespalib::nbostream &out);
-
- /*
- * To be called after earlyOpen() but before afterOpen().
- */
- virtual void
- checkPointRead(vespalib::nbostream &in);
-
virtual void
setFeatureParams(const PostingListParams &params);
diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp
index 9f03c06cf56..8b0c60d0bd7 100644
--- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp
@@ -5,14 +5,12 @@
#include "extposocc.h"
#include "pagedict4file.h"
#include <vespa/vespalib/util/error.h>
-#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/log/log.h>
LOG_SETUP(".diskindex.fieldwriter");
namespace search::diskindex {
-using vespalib::nbostream;
using vespalib::getLastErrorString;
using common::FileHeaderContext;
@@ -181,36 +179,6 @@ FieldWriter::close()
void
-FieldWriter::checkPointWrite(nbostream &out)
-{
- out << _wordNum << _prevDocId;
- out << _docIdLimit << _numWordIds;
- out << _compactWordNum << _word;
- _posoccfile->checkPointWrite(out);
- _dictFile->checkPointWrite(out);
- _bvc.checkPointWrite(out);
- _bmapfile.checkPointWrite(out);
-}
-
-
-void
-FieldWriter::checkPointRead(nbostream &in)
-{
- in >> _wordNum >> _prevDocId;
- uint32_t checkDocIdLimit = 0;
- uint64_t checkNumWordIds = 0;
- in >> checkDocIdLimit >> checkNumWordIds;
- assert(checkDocIdLimit == _docIdLimit);
- assert(checkNumWordIds == _numWordIds);
- in >> _compactWordNum >> _word;
- _posoccfile->checkPointRead(in);
- _dictFile->checkPointRead(in);
- _bvc.checkPointRead(in);
- _bmapfile.checkPointRead(in);
-}
-
-
-void
FieldWriter::setFeatureParams(const PostingListParams &params)
{
_posoccfile->setFeatureParams(params);
diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h
index 1cd53b99bce..893572fbdad 100644
--- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h
+++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h
@@ -8,8 +8,6 @@
#include <vespa/searchlib/bitcompression/countcompression.h>
#include <vespa/searchlib/bitcompression/posocccompression.h>
-namespace vespalib { class nbostream; }
-
namespace search {
namespace diskindex {
@@ -82,15 +80,6 @@ public:
bool close();
- /*
- * To be called between words, not in the middle of one.
- */
- void checkPointWrite(vespalib::nbostream &out);
-
- /*
- * To be called after earlyOpen() but before afterOpen().
- */
- void checkPointRead(vespalib::nbostream &in);
void setFeatureParams(const PostingListParams &params);
void getFeatureParams(PostingListParams &params);
static void remove(const vespalib::string &prefix);
diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
index 97d21caef96..8d6b9d04b51 100644
--- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
@@ -321,7 +321,6 @@ Fusion::openFieldWriter(const SchemaUtil::IndexIterator &index,
index.getSchema(),
index.getIndex(),
_tuneFileIndexing._write);
- // No checkpointing
if (!writer.lateOpen(_tuneFileIndexing._write,
_fileHeaderContext)) {
LOG(error, "Could not open output posocc + dictionary in %s",
diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp
index aacfa400755..0fdc4b96202 100644
--- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp
@@ -296,8 +296,6 @@ FileHandle::open(const vespalib::stringref &dir,
index.getSchema(), index.getIndex(),
tuneFileWrite);
- // No checkpointing
-
if (!_fieldWriter->lateOpen(tuneFileWrite, fileHeaderContext)) {
LOG(error, "Could not open term writer %s for write (%s)",
dir.c_str(), getLastErrorString().c_str());
diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp
index 9c63c5930b6..72fd07e5752 100644
--- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp
@@ -4,7 +4,6 @@
#include <vespa/searchlib/common/fileheadercontext.h>
#include <vespa/vespalib/stllike/asciistream.h>
#include <vespa/vespalib/data/fileheader.h>
-#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/io/fileutil.h>
#include <vespa/log/log.h>
@@ -63,8 +62,7 @@ PageDict4FileSeqRead::PageDict4FileSeqRead()
_ssCompleted(false),
_spCompleted(false),
_pCompleted(false),
- _wordNum(0u),
- _checkPointData()
+ _wordNum(0u)
{
_ssd.setReadContext(&_ssReadContext);
_spd.setReadContext(&_spReadContext);
@@ -229,26 +227,13 @@ PageDict4FileSeqRead::open(const vespalib::string &name,
32768u);
_ssd.emptyBuffer(0);
- if (_checkPointData) {
- _ssReadContext.setPosition(_ssReadContext.getCheckPointOffset());
- if (_ssd._valI >= _ssd._valE)
- _ssReadContext.readComprBuffer();
- _spReadContext.setPosition(_spReadContext.getCheckPointOffset());
- if (_spd._valI >= _spd._valE)
- _spReadContext.readComprBuffer();
- _pReadContext.setPosition(_pReadContext.getCheckPointOffset());
- if (_pd._valI >= _pd._valE)
- _pReadContext.readComprBuffer();
- } else {
- _ssReadContext.readComprBuffer();
- assert(_ssReadContext.getBufferEndFilePos() >= fileSize);
- readSSHeader();
- _spReadContext.readComprBuffer();
- readSPHeader();
- _pReadContext.readComprBuffer();
- readPHeader();
- }
-
+ _ssReadContext.readComprBuffer();
+ assert(_ssReadContext.getBufferEndFilePos() >= fileSize);
+ readSSHeader();
+ _spReadContext.readComprBuffer();
+ readSPHeader();
+ _pReadContext.readComprBuffer();
+ readPHeader();
_ssReader = new SSReader(_ssReadContext,
_ssHeaderLen,
@@ -263,15 +248,9 @@ PageDict4FileSeqRead::open(const vespalib::string &name,
_spd,
_pd);
- if (_checkPointData) {
- _ssReader->checkPointRead(*_checkPointData);
- _pReader->checkPointRead(*_checkPointData);
- assert(_checkPointData->empty());
- } else {
- _ssReader->setup(_ssd);
- _pReader->setup();
- _wordNum = 0;
- }
+ _ssReader->setup(_ssd);
+ _pReader->setup();
+ _wordNum = 0;
return true;
}
@@ -299,40 +278,6 @@ PageDict4FileSeqRead::close()
void
-PageDict4FileSeqRead::checkPointWrite(vespalib::nbostream &out)
-{
- _ssd.checkPointWrite(out);
- _spReadContext.checkPointWrite(out);
- _pReadContext.checkPointWrite(out);
- vespalib::nbostream data;
- _ssReader->checkPointWrite(data);
- _pReader->checkPointWrite(data);
- std::vector<char> checkPointData(data.size());
- data.read(&checkPointData[0], data.size());
- out << checkPointData;
- out << _wordNum;
- out << _ssCompleted << _ssFileBitSize << _ssHeaderLen;
- out << _spCompleted << _spFileBitSize << _spHeaderLen;
- out << _pCompleted << _pFileBitSize << _pHeaderLen;
-}
-
-void
-PageDict4FileSeqRead::checkPointRead(vespalib::nbostream &in)
-{
- _ssd.checkPointRead(in);
- _spReadContext.checkPointRead(in);
- _pReadContext.checkPointRead(in);
- std::vector<char> checkPointData;
- in >> checkPointData;
- _checkPointData.reset(new vespalib::nbostream(checkPointData.size()));
- _checkPointData->write(&checkPointData[0], checkPointData.size());
- in >> _wordNum;
- in >> _ssCompleted >> _ssFileBitSize >> _ssHeaderLen;
- in >> _spCompleted >> _spFileBitSize >> _spHeaderLen;
- in >> _pCompleted >> _pFileBitSize >> _pHeaderLen;
-}
-
-void
PageDict4FileSeqRead::getParams(PostingListParams &params)
{
params.clear();
@@ -418,11 +363,9 @@ PageDict4FileSeqWrite::open(const vespalib::string &name,
assertOpenWriteOnly(ok, ssname);
_ssWriteContext.setFile(&_ssfile);
- if (!_checkPointData) {
- _pWriteContext.allocComprBuf(65536u, 32768u);
- _spWriteContext.allocComprBuf(65536u, 32768u);
- _ssWriteContext.allocComprBuf(65536u, 32768u);
- }
+ _pWriteContext.allocComprBuf(65536u, 32768u);
+ _spWriteContext.allocComprBuf(65536u, 32768u);
+ _ssWriteContext.allocComprBuf(65536u, 32768u);
uint64_t pFileSize = _pfile.GetSize();
uint64_t spFileSize = _spfile.GetSize();
@@ -445,33 +388,24 @@ PageDict4FileSeqWrite::open(const vespalib::string &name,
assert(ssBufferStartFilePos ==
static_cast<uint64_t>(_ssfile.GetPosition()));
- if (!_checkPointData) {
- _pe.setupWrite(_pWriteContext);
- _spe.setupWrite(_spWriteContext);
- _sse.setupWrite(_ssWriteContext);
- assert(_pe.getWriteOffset() == 0);
- assert(_spe.getWriteOffset() == 0);
- assert(_sse.getWriteOffset() == 0);
- _spe.copyParams(_sse);
- _pe.copyParams(_sse);
- // Write initial file headers
- makePHeader(fileHeaderContext);
- makeSPHeader(fileHeaderContext);
- makeSSHeader(fileHeaderContext);
- }
+ _pe.setupWrite(_pWriteContext);
+ _spe.setupWrite(_spWriteContext);
+ _sse.setupWrite(_ssWriteContext);
+ assert(_pe.getWriteOffset() == 0);
+ assert(_spe.getWriteOffset() == 0);
+ assert(_sse.getWriteOffset() == 0);
+ _spe.copyParams(_sse);
+ _pe.copyParams(_sse);
+ // Write initial file headers
+ makePHeader(fileHeaderContext);
+ makeSPHeader(fileHeaderContext);
+ makeSSHeader(fileHeaderContext);
_ssWriter = new SSWriter(_sse);
_spWriter = new SPWriter(*_ssWriter, _spe);
_pWriter = new PWriter(*_spWriter, _pe);
- if (_checkPointData) {
- _ssWriter->checkPointRead(*_checkPointData);
- _spWriter->checkPointRead(*_checkPointData);
- _pWriter->checkPointRead(*_checkPointData);
- assert(_checkPointData->empty());
- } else {
- _spWriter->setup();
- _pWriter->setup();
- }
+ _spWriter->setup();
+ _pWriter->setup();
return true;
}
@@ -683,43 +617,6 @@ PageDict4FileSeqWrite::updateSSHeader(uint64_t fileBitSize)
void
-PageDict4FileSeqWrite::checkPointWrite(vespalib::nbostream &out)
-{
- _ssWriteContext.writeComprBuffer(true);
- _spWriteContext.writeComprBuffer(true);
- _pWriteContext.writeComprBuffer(true);
- _ssWriteContext.checkPointWrite(out);
- _spWriteContext.checkPointWrite(out);
- _pWriteContext.checkPointWrite(out);
- vespalib::nbostream data;
- _ssWriter->checkPointWrite(data);
- _spWriter->checkPointWrite(data);
- _pWriter->checkPointWrite(data);
- std::vector<char> checkPointData(data.size());
- data.read(&checkPointData[0], data.size());
- out << checkPointData;
- out << _ssHeaderLen << _spHeaderLen << _pHeaderLen;
- _ssfile.Sync();
- _spfile.Sync();
- _pfile.Sync();
-}
-
-
-void
-PageDict4FileSeqWrite::checkPointRead(vespalib::nbostream &in)
-{
- _ssWriteContext.checkPointRead(in);
- _spWriteContext.checkPointRead(in);
- _pWriteContext.checkPointRead(in);
- std::vector<char> checkPointData;
- in >> checkPointData;
- _checkPointData.reset(new vespalib::nbostream(checkPointData.size()));
- _checkPointData->write(&checkPointData[0], checkPointData.size());
- in >> _ssHeaderLen >> _spHeaderLen >> _pHeaderLen;
-}
-
-
-void
PageDict4FileSeqWrite::setParams(const PostingListParams &params)
{
params.get("avgBitsPerDoc", _sse._avgBitsPerDoc);
diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h
index fe432a22463..7baa883a740 100644
--- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h
+++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h
@@ -50,8 +50,6 @@ class PageDict4FileSeqRead : public index::DictionaryFileSeqRead
uint64_t _wordNum;
- std::unique_ptr<vespalib::nbostream> _checkPointData;
-
void
readSSHeader();
@@ -85,21 +83,6 @@ public:
*/
virtual bool close() override;
- /**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt. Implies
- * flush from memory to disk, and possibly also sync to permanent
- * storage media.
- */
- virtual void
- checkPointWrite(vespalib::nbostream &out) override;
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- virtual void
- checkPointRead(vespalib::nbostream &in) override;
-
/*
* Get current parameters.
*/
@@ -142,8 +125,6 @@ class PageDict4FileSeqWrite : public index::DictionaryFileSeqWrite
uint32_t _spHeaderLen; // Length of header for sparse page file (bytes)
uint32_t _ssHeaderLen; // Length of header for sparse sparse file (bytes)
- std::unique_ptr<vespalib::nbostream> _checkPointData;
-
void
writeIndexNames(vespalib::GenericHeader &header);
@@ -196,21 +177,6 @@ public:
virtual bool
close() override;
- /**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt. Implies
- * flush from memory to disk, and possibly also sync to permanent
- * storage media.
- */
- virtual void
- checkPointWrite(vespalib::nbostream &out) override;
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- virtual void
- checkPointRead(vespalib::nbostream &in) override;
-
/*
* Set parameters.
*/
diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp
index 768ee972097..17d4457a318 100644
--- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp
@@ -6,7 +6,6 @@
#include <vespa/searchlib/index/postinglistfile.h>
#include <vespa/searchlib/index/docidandfeatures.h>
#include <vespa/searchlib/common/fileheadercontext.h>
-#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/data/fileheader.h>
#include <vespa/log/log.h>
@@ -27,7 +26,6 @@ using index::PostingListCountFileSeqWrite;
using common::FileHeaderContext;
using bitcompression::FeatureDecodeContextBE;
using bitcompression::FeatureEncodeContextBE;
-using vespalib::nbostream;
using vespalib::getLastErrorString;
@@ -78,11 +76,7 @@ Zc4PostingSeqRead(PostingListCountFileSeqRead *countFile)
_rangeEndOffset(0),
_readAheadEndOffset(0),
_wordStart(0),
- _checkPointPos(0),
- _residue(0),
- _checkPointChunkNo(0u),
- _checkPointResidue(0u),
- _checkPointHasMore(false)
+ _residue(0)
{
if (_countFile != NULL) {
PostingListParams params;
@@ -231,32 +225,6 @@ readDocIdAndFeatures(DocIdAndFeatures &features)
void
-Zc4PostingSeqRead::checkPointWrite(nbostream &out)
-{
- out << _counts;
- out << _wordStart;
- uint64_t curPos = _decodeContext->getReadOffset();
- out << curPos;
- out << _residue;
- out << _chunkNo;
- out << _hasMore;
-}
-
-
-void
-Zc4PostingSeqRead::checkPointRead(nbostream &in)
-{
- in >> _counts;
- in >> _wordStart;
- in >> _checkPointPos;
- in >> _checkPointResidue;
- in >> _checkPointChunkNo;
- in >> _checkPointHasMore;
- assert(_checkPointPos >= _wordStart);
-}
-
-
-void
Zc4PostingSeqRead::readWordStartWithSkip()
{
typedef FeatureEncodeContextBE EC;
@@ -508,34 +476,8 @@ Zc4PostingSeqRead::open(const vespalib::string &name,
_readContext.readComprBuffer();
readHeader();
- if (d._valI >= d._valE)
+ if (d._valI >= d._valE) {
_readContext.readComprBuffer();
- if (_checkPointPos != 0) {
- if (_checkPointResidue != 0 || _checkPointHasMore) {
- // Checkpointed in the middle of a word. Read from
- // start at word until at right position.
- DocIdAndFeatures features;
- _readContext.setPosition(_wordStart);
- assert(_decodeContext->getReadOffset() == _wordStart);
- _readContext.readComprBuffer();
- readWordStart();
- assert(_chunkNo < _checkPointChunkNo ||
- (_chunkNo == _checkPointChunkNo &&
- _residue >= _checkPointResidue));
- while (_chunkNo < _checkPointChunkNo ||
- _residue > _checkPointResidue) {
- readDocIdAndFeatures(features);
- }
- assert(_chunkNo == _checkPointChunkNo);
- assert(_residue == _checkPointResidue);
- assert(_hasMore == _checkPointHasMore);
- assert(_decodeContext->getReadOffset() == _checkPointPos);
- } else {
- // Checkpointed between words.
- _readContext.setPosition(_checkPointPos);
- assert(_decodeContext->getReadOffset() == _checkPointPos);
- _readContext.readComprBuffer();
- }
}
} else {
LOG(error, "could not open %s: %s",
@@ -740,29 +682,6 @@ Zc4PostingSeqWrite::flushWord()
}
-void
-Zc4PostingSeqWrite::checkPointWrite(nbostream &out)
-{
- _writeContext.writeComprBuffer(true); // Also flush slack
- out << _numWords;
- _writeContext.checkPointWrite(out);
- _featureWriteContext.checkPointWrite(out);
- out.saveVector(_docIds) << _writePos << _counts;
- _file.Sync();
-}
-
-
-void
-Zc4PostingSeqWrite::checkPointRead(nbostream &in)
-{
- in >> _numWords;
- _writeContext.checkPointRead(in);
- _featureWriteContext.checkPointRead(in);
- _featureOffset = _encodeFeatures->getWriteOffset();
- in.restoreVector(_docIds) >> _writePos >> _counts;
-}
-
-
uint32_t
Zc4PostingSeqWrite::readHeader(const vespalib::string &name)
{
@@ -1422,10 +1341,3 @@ ZcPostingSeqWrite::flushWordNoSkip()
}
} // namespace search::diskindex
-
-#include <vespa/vespalib/objects/nbostream.hpp>
-namespace vespalib {
- using UInt32Pair = std::pair<uint32_t, uint32_t>;
- template nbostream &nbostream::saveVector<UInt32Pair>(const std::vector<UInt32Pair> &);
- template nbostream &nbostream::restoreVector<UInt32Pair>(std::vector<UInt32Pair> &);
-}
diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.h b/searchlib/src/vespa/searchlib/diskindex/zcposting.h
index 64879aa05fb..9a650abdd40 100644
--- a/searchlib/src/vespa/searchlib/diskindex/zcposting.h
+++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.h
@@ -78,11 +78,7 @@ protected:
uint64_t _rangeEndOffset; // End offset for word pair
uint64_t _readAheadEndOffset; // Readahead end offset for word pair
uint64_t _wordStart; // last word header position
- uint64_t _checkPointPos; // file position when checkpointing
uint32_t _residue; // Number of unread documents after word header
- uint32_t _checkPointChunkNo; // _chunkNo when checkpointing
- uint32_t _checkPointResidue; // _residue when checkpointing
- bool _checkPointHasMore; // _hasMore when checkpointing
public:
Zc4PostingSeqRead(index::PostingListCountFileSeqRead *countFile);
@@ -98,8 +94,6 @@ public:
virtual void readCommonWordDocIdAndFeatures(DocIdAndFeatures &features);
void readDocIdAndFeatures(DocIdAndFeatures &features) override;
- void checkPointWrite(vespalib::nbostream &out) override;
- void checkPointRead(vespalib::nbostream &in) override;
void readCounts(const PostingListCounts &counts) override; // Fill in for next word
bool open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) override;
bool close() override;
@@ -176,8 +170,6 @@ public:
void writeDocIdAndFeatures(const DocIdAndFeatures &features) override;
void flushWord() override;
- void checkPointWrite(vespalib::nbostream &out) override;
- void checkPointRead(vespalib::nbostream &in) override;
bool open(const vespalib::string &name, const TuneFileSeqWrite &tuneFileWrite,
const search::common::FileHeaderContext &fileHeaderContext) override;
diff --git a/searchlib/src/vespa/searchlib/index/postinglistcountfile.h b/searchlib/src/vespa/searchlib/index/postinglistcountfile.h
index 5b14b477c42..30c6963f228 100644
--- a/searchlib/src/vespa/searchlib/index/postinglistcountfile.h
+++ b/searchlib/src/vespa/searchlib/index/postinglistcountfile.h
@@ -5,8 +5,6 @@
#include "postinglistcounts.h"
#include <vespa/searchlib/common/tunefileinfo.h>
-namespace vespalib { class nbostream; }
-
namespace search {
namespace common { class FileHeaderContext; }
@@ -34,19 +32,6 @@ public:
virtual ~PostingListCountFileSeqRead();
/**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt. Implies
- * flush from memory to disk, and possibly also sync to permanent
- * storage media.
- */
- virtual void checkPointWrite(vespalib::nbostream &out) = 0;
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- virtual void checkPointRead(vespalib::nbostream &in) = 0;
-
- /**
* Open posting list count file for sequential read.
*/
virtual bool open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) = 0;
@@ -71,19 +56,6 @@ public:
virtual ~PostingListCountFileSeqWrite();
/**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt. Implies
- * flush from memory to disk, and possibly also sync to permanent
- * storage media.
- */
- virtual void checkPointWrite(vespalib::nbostream &out) = 0;
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- virtual void checkPointRead(vespalib::nbostream &in) = 0;
-
- /**
* Open posting list count file for sequential write.
*/
virtual bool open(const vespalib::string &name,
diff --git a/searchlib/src/vespa/searchlib/index/postinglistfile.h b/searchlib/src/vespa/searchlib/index/postinglistfile.h
index 43628bd798b..61247ef3197 100644
--- a/searchlib/src/vespa/searchlib/index/postinglistfile.h
+++ b/searchlib/src/vespa/searchlib/index/postinglistfile.h
@@ -8,8 +8,6 @@
class FastOS_FileInterface;
-namespace vespalib { class nbostream; }
-
namespace search {
namespace common { class FileHeaderContext; }
@@ -38,19 +36,6 @@ public:
virtual void readDocIdAndFeatures(DocIdAndFeatures &features) = 0;
/**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt. Implies
- * flush from memory to disk, and possibly also sync to permanent
- * storage media.
- */
- virtual void checkPointWrite(vespalib::nbostream &out) = 0;
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- virtual void checkPointRead(vespalib::nbostream &in) = 0;
-
- /**
* Read counts for a word.
*/
virtual void readCounts(const PostingListCounts &counts) = 0;
@@ -135,19 +120,6 @@ public:
virtual void flushWord() = 0;
/**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt. Implies
- * flush from memory to disk, and possibly also sync to permanent
- * storage media.
- */
- virtual void checkPointWrite(vespalib::nbostream &out) = 0;
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- virtual void checkPointRead(vespalib::nbostream &in) = 0;
-
- /**
* Open posting list file for sequential write.
*/
virtual bool
diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp
index 8c7f97db022..acfc2e46020 100644
--- a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp
+++ b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp
@@ -1,14 +1,12 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "comprbuffer.h"
-#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/fastos/file.h>
#include <cassert>
+#include <cstring>
namespace search {
-using vespalib::nbostream;
-
ComprBuffer::ComprBuffer(uint32_t unitSize)
: _comprBuf(NULL),
_comprBufSize(0),
@@ -114,25 +112,4 @@ ComprBuffer::referenceComprBuf(const ComprBuffer &rhs)
_comprBufSize = rhs._comprBufSize;
}
-
-void
-ComprBuffer::checkPointWrite(nbostream &out)
-{
- _aligner.checkPointWrite(out);
- out << _comprBufSize << _unitSize << _padBefore;
-}
-
-
-void
-ComprBuffer::checkPointRead(nbostream &in)
-{
- _aligner.checkPointRead(in);
- uint32_t unitSize;
- in >> _comprBufSize >> unitSize >> _padBefore;
- assert(unitSize == _unitSize);
-
- allocComprBuf();
-}
-
-
}
diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.h b/searchlib/src/vespa/searchlib/util/comprbuffer.h
index 61928976acc..c15671d5e14 100644
--- a/searchlib/src/vespa/searchlib/util/comprbuffer.h
+++ b/searchlib/src/vespa/searchlib/util/comprbuffer.h
@@ -31,7 +31,6 @@ public:
static size_t minimumPadding() { return 8; }
uint32_t getUnitBitSize() const { return _unitSize * 8; }
bool getPadBefore() const { return _padBefore; }
- bool getCheckPointResumed() const { return _aligner.getCheckPointResumed(); }
/*
* When encoding to memory instead of file, the compressed buffer must
@@ -44,18 +43,6 @@ public:
* long as rhs is live and unchanged.
*/
void referenceComprBuf(const ComprBuffer &rhs);
-
- /**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt.
- */
- void checkPointWrite(vespalib::nbostream &out);
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- *
- */
- void checkPointRead(vespalib::nbostream &in);
};
}
diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp
index 8b17fd85753..26c66f43993 100644
--- a/searchlib/src/vespa/searchlib/util/comprfile.cpp
+++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp
@@ -1,14 +1,12 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "comprfile.h"
-#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/fastos/file.h>
#include <cassert>
+#include <cstring>
namespace search {
-using vespalib::nbostream;
-
void
ComprFileReadBase::ReadComprBuffer(uint64_t stopOffset,
bool readAll,
@@ -292,9 +290,7 @@ ComprFileReadContext(ComprFileDecodeContext &decodeContext)
_bitOffset(0),
_stopOffset(0),
_readAll(true),
- _checkPointOffsetValid(false),
- _file(NULL),
- _checkPointOffset(0)
+ _file(NULL)
{
}
@@ -308,9 +304,7 @@ ComprFileReadContext(uint32_t unitSize)
_bitOffset(0),
_stopOffset(0),
_readAll(true),
- _checkPointOffsetValid(false),
- _file(NULL),
- _checkPointOffset(0)
+ _file(NULL)
{
}
@@ -497,28 +491,6 @@ ComprFileReadContext::copyReadContext(const ComprFileReadContext &rhs)
}
}
-
-void
-ComprFileReadContext::checkPointWrite(nbostream &out)
-{
- ComprBuffer::checkPointWrite(out);
- ComprFileDecodeContext &d = *_decodeContext;
- d.checkPointWrite(out);
- uint64_t bitOffset = d.getBitPosV();
- out << bitOffset;
-}
-
-
-void
-ComprFileReadContext::checkPointRead(nbostream &in)
-{
- ComprBuffer::checkPointRead(in);
- ComprFileDecodeContext &d = *_decodeContext;
- d.checkPointRead(in);
- in >> _checkPointOffset; // Cannot seek until file is opened
- _checkPointOffsetValid = true;
-}
-
ComprFileWriteContext::
ComprFileWriteContext(ComprFileEncodeContext &encodeContext)
: ComprBuffer(encodeContext.getUnitByteSize()),
@@ -598,47 +570,4 @@ ComprFileWriteContext::allocComprBuf()
allocComprBuf(32768, 32768);
}
-
-void
-ComprFileWriteContext::checkPointWrite(nbostream &out)
-{
- ComprBuffer::checkPointWrite(out);
- ComprFileEncodeContext &e = *_encodeContext;
- uint64_t bufferStartFilePos = getBufferStartFilePos();
- uint64_t usedSize = e.getUsedUnits(_comprBuf) *
- e.getUnitByteSize();
- out << bufferStartFilePos << usedSize;
- e.checkPointWrite(out);
- if (usedSize != 0) {
- out.write(_comprBuf, usedSize);
- }
- uint64_t bitOffset = e.getBitPosV();
- out << bitOffset;
-}
-
-
-void
-ComprFileWriteContext::checkPointRead(nbostream &in)
-{
- ComprBuffer::checkPointRead(in);
- ComprFileEncodeContext &e = *_encodeContext;
- uint64_t bufferStartFilePos = 0;
- uint64_t usedSize = 0;
- in >> bufferStartFilePos >> usedSize;
- e.checkPointRead(in);
- if (usedSize != 0) {
- assert((usedSize % e.getUnitByteSize()) == 0);
- assert(_comprBufSize >= usedSize / e.getUnitByteSize());
- in.read(_comprBuf, usedSize);
- }
- setBufferStartFilePos(bufferStartFilePos);
- e.afterWrite(*this, usedSize / e.getUnitByteSize(), bufferStartFilePos);
- uint64_t bitOffset = 0;
- in >> bitOffset;
- uint64_t writeOffset = e.getBitPosV();
- assert(bitOffset == writeOffset);
- (void) writeOffset;
-}
-
-
}
diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h
index b69ef140868..3d44f088c74 100644
--- a/searchlib/src/vespa/searchlib/util/comprfile.h
+++ b/searchlib/src/vespa/searchlib/util/comprfile.h
@@ -7,8 +7,6 @@
class FastOS_FileInterface;
-namespace vespalib { class nbostream; }
-
namespace search {
class ComprFileWriteContext;
@@ -60,19 +58,6 @@ public:
* Get size of each unit (typically 4 or 8)
*/
virtual uint32_t getUnitByteSize() const = 0;
-
- /**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt. Caller must
- * save position.
- */
- virtual void checkPointWrite(vespalib::nbostream &out) = 0;
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- * Caller must restore position.
- */
- virtual void checkPointRead(vespalib::nbostream &in) = 0;
};
class ComprFileReadBase
@@ -110,9 +95,7 @@ private:
int _bitOffset;
uint64_t _stopOffset;
bool _readAll;
- bool _checkPointOffsetValid; // Set only if checkpoint has been read
FastOS_FileInterface *_file;
- uint64_t _checkPointOffset; // bit offset saved by checkPointRead
public:
ComprFileReadContext(ComprFileDecodeContext &decodeContext);
@@ -170,19 +153,6 @@ public:
* For unit testing only. Copy data owned by rhs.
*/
void copyWriteContext(const ComprFileWriteContext &rhs);
-
- /**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt.
- */
- void checkPointWrite(vespalib::nbostream &out);
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- void checkPointRead(vespalib::nbostream &in);
- bool getCheckPointOffsetValid() const { return _checkPointOffsetValid; }
- uint64_t getCheckPointOffset() const { return _checkPointOffset; }
};
@@ -216,19 +186,6 @@ public:
*/
virtual uint32_t getUnitByteSize() const = 0;
- /**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt. Caller must
- * save position, although partial unit is saved.
- */
- virtual void checkPointWrite(vespalib::nbostream &out) = 0;
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- * Caller must restore positon, although partial unit is restored.
- */
- virtual void checkPointRead(vespalib::nbostream &in) = 0;
-
virtual uint64_t getBitPosV() const = 0;
};
@@ -281,17 +238,6 @@ public:
* no file is attached.
*/
std::pair<void *, size_t> grabComprBuffer(void *&comprBufMalloc);
-
- /**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt.
- */
- void checkPointWrite(vespalib::nbostream &out);
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- void checkPointRead(vespalib::nbostream &in);
};
}
diff --git a/searchlib/src/vespa/searchlib/util/filealign.cpp b/searchlib/src/vespa/searchlib/util/filealign.cpp
index 15bef611714..b60c0492854 100644
--- a/searchlib/src/vespa/searchlib/util/filealign.cpp
+++ b/searchlib/src/vespa/searchlib/util/filealign.cpp
@@ -1,14 +1,11 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "filealign.h"
-#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/fastos/file.h>
#include <cassert>
namespace search {
-using vespalib::nbostream;
-
namespace {
size_t
@@ -42,8 +39,7 @@ FileAlign::FileAlign()
_minAlignedSize(1),
_elemSize(1),
_directIOMemAlign(1),
- _directio(false),
- _checkPointResumed(false)
+ _directio(false)
{ }
@@ -116,24 +112,4 @@ FileAlign::setupAlign(size_t elements,
return elements;
}
-
-void
-FileAlign::checkPointWrite(nbostream &out)
-{
- out << _directIOFileAlign << _preferredFileAlign <<
- _minDirectIOSize << _minAlignedSize << _elemSize <<
- _directIOMemAlign << _directio;
-}
-
-
-void
-FileAlign::checkPointRead(nbostream &in)
-{
- in >> _directIOFileAlign >> _preferredFileAlign >>
- _minDirectIOSize >> _minAlignedSize >> _elemSize >>
- _directIOMemAlign >> _directio;
- _checkPointResumed = true;
-}
-
-
}
diff --git a/searchlib/src/vespa/searchlib/util/filealign.h b/searchlib/src/vespa/searchlib/util/filealign.h
index 3e75fb2f371..49d053957ce 100644
--- a/searchlib/src/vespa/searchlib/util/filealign.h
+++ b/searchlib/src/vespa/searchlib/util/filealign.h
@@ -7,8 +7,6 @@
class FastOS_FileInterface;
-namespace vespalib { class nbostream; }
-
namespace search {
class FileAlign
@@ -21,7 +19,6 @@ private:
size_t _elemSize;
size_t _directIOMemAlign;
bool _directio;
- bool _checkPointResumed;
public:
@@ -64,24 +61,12 @@ public:
*/
size_t setupAlign(size_t elements, size_t elemSize, FastOS_FileInterface *file, size_t preferredFileAlignment);
bool getDirectIO() const { return _directio; }
- bool getCheckPointResumed() const { return _checkPointResumed; }
size_t getDirectIOFileAlign() const { return _directIOFileAlign; }
size_t getDirectIOMemAlign() const { return _directIOMemAlign; }
size_t getMinDirectIOSize() const { return _minDirectIOSize; }
size_t getMinAlignedSize() const { return _minAlignedSize; }
size_t getPreferredFileAlign() const { return _preferredFileAlign; }
size_t getElemSize() const { return _elemSize; }
-
- /**
- * Checkpoint write. Used at semi-regular intervals during indexing
- * to allow for continued indexing after an interrupt.
- */
- void checkPointWrite(vespalib::nbostream &out);
-
- /**
- * Checkpoint read. Used when resuming indexing after an interrupt.
- */
- void checkPointRead(vespalib::nbostream &in);
};
}