diff options
author | Tor Egge <Tor.Egge@oath.com> | 2017-10-09 19:01:30 +0000 |
---|---|---|
committer | Tor Egge <Tor.Egge@oath.com> | 2017-10-09 19:01:30 +0000 |
commit | c67bff3e28796376ee19f66edb070aec1b540777 (patch) | |
tree | b38167526c092ba2b9a19bb27e3d35ffb4a9e033 | |
parent | f2cd4cf637ae433341e6410145d3590e206889d3 (diff) |
Eliminate split open handling (earlyOpen() + lateOpen()), use open() instead.
10 files changed, 52 insertions, 161 deletions
diff --git a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp index 223f224aba5..dc352f70706 100644 --- a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp @@ -47,8 +47,7 @@ FieldWriterWrapper::open(const std::string &path, const common::FileHeaderContext &fileHeaderContext) { vespalib::mkdir(path, false); - _writer.earlyOpen(path, 64, 10000, false, schema, indexId, tuneFileWrite); - return _writer.lateOpen(tuneFileWrite, fileHeaderContext); + return _writer.open(path, 64, 10000, false, schema, indexId, tuneFileWrite, fileHeaderContext); } FieldWriterWrapper & diff --git a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp index 0746f62b1df..71467519dbd 100644 --- a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp +++ b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp @@ -87,9 +87,6 @@ makeWordString(uint64_t wordNum) } -typedef std::shared_ptr<FieldReader> FieldReaderSP; -typedef std::shared_ptr<FieldWriter> FieldWriterSP; - class FieldWriterTest : public FastOS_Application { private: @@ -143,7 +140,7 @@ FieldWriterTest::~FieldWriterTest() class WrappedFieldWriter { public: - FieldWriterSP _fieldWriter; + std::unique_ptr<FieldWriter> _fieldWriter; private: bool _dynamicK; uint32_t _numWordIds; @@ -160,8 +157,6 @@ public: uint32_t docIdLimit); ~WrappedFieldWriter(); - void earlyOpen(); - void lateOpen(); void open(); void close(); }; @@ -187,32 +182,16 @@ WrappedFieldWriter::WrappedFieldWriter(const vespalib::string &namepref, void -WrappedFieldWriter::earlyOpen() -{ - TuneFileSeqWrite tuneFileWrite; - _fieldWriter.reset(new FieldWriter(_docIdLimit, _numWordIds)); - _fieldWriter->earlyOpen(_namepref, - minSkipDocs, minChunkDocs, _dynamicK, _schema, - _indexId, - tuneFileWrite); -} - - -void -WrappedFieldWriter::lateOpen() +WrappedFieldWriter::open() { TuneFileSeqWrite tuneFileWrite; DummyFileHeaderContext fileHeaderContext; fileHeaderContext.disableFileName(); - _fieldWriter->lateOpen(tuneFileWrite, fileHeaderContext); -} - - -void -WrappedFieldWriter::open() -{ - earlyOpen(); - lateOpen(); + _fieldWriter = std::make_unique<FieldWriter>(_docIdLimit, _numWordIds); + _fieldWriter->open(_namepref, + minSkipDocs, minChunkDocs, _dynamicK, _schema, + _indexId, + tuneFileWrite, fileHeaderContext); } @@ -227,7 +206,7 @@ WrappedFieldWriter::close() class WrappedFieldReader { public: - FieldReaderSP _fieldReader; + std::unique_ptr<FieldReader> _fieldReader; private: std::string _namepref; uint32_t _numWordIds; @@ -243,8 +222,6 @@ public: uint32_t docIdLimit); ~WrappedFieldReader(); - void earlyOpen(); - void lateOpen(); void open(); void close(); }; @@ -276,35 +253,17 @@ WrappedFieldReader::~WrappedFieldReader() { } - void -WrappedFieldReader::earlyOpen() -{ - TuneFileSeqRead tuneFileRead; - _fieldReader.reset(new FieldReader()); - _fieldReader->earlyOpen(_namepref, tuneFileRead); -} - - -void -WrappedFieldReader::lateOpen() +WrappedFieldReader::open() { TuneFileSeqRead tuneFileRead; _wmap.setup(_numWordIds); _dmap.setup(_docIdLimit); + _fieldReader = std::make_unique<FieldReader>(); _fieldReader->setup(_wmap, _dmap); - _fieldReader->lateOpen(_namepref, tuneFileRead); + _fieldReader->open(_namepref, tuneFileRead); } - -void -WrappedFieldReader::open() -{ - earlyOpen(); - lateOpen(); -} - - void WrappedFieldReader::close() { @@ -343,7 +302,7 @@ writeField(FakeWordSet &wordSet, for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) { FakeWord &fw = *wordSet._words[wc][wi]; ostate._fieldWriter->newWord(makeWordString(wordNum)); - fw.dump(ostate._fieldWriter, false); + fw.dump(*ostate._fieldWriter, false); ++wordNum; } } @@ -396,7 +355,7 @@ readField(FakeWordSet &wordSet, TermFieldMatchDataArray tfmda; tfmda.add(&mdfield1); - fw.validate(istate._fieldReader, wordNum, + fw.validate(*istate._fieldReader, wordNum, tfmda, verbose); ++wordNum; } diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp index 108defea6a0..cffc2e09ef8 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp @@ -106,8 +106,8 @@ FieldReader::setup(const WordNumMapping &wordNumMapping, bool -FieldReader::earlyOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) +FieldReader::open(const vespalib::string &prefix, + const TuneFileSeqRead &tuneFileRead) { vespalib::string name = prefix + "posocc.dat.compressed"; FastOS_StatInfo statInfo; @@ -122,28 +122,20 @@ FieldReader::earlyOpen(const vespalib::string &prefix, return false; } - _dictFile.reset(new search::diskindex::PageDict4FileSeqRead); + _dictFile = std::make_unique<PageDict4FileSeqRead>(); PostingListParams featureParams; - _oldposoccfile.reset(search::diskindex::makePosOccRead(name, - _dictFile.get(), - dynamicKPosOccFormat, - featureParams, - tuneFileRead)); - return true; -} - - -bool -FieldReader::lateOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) -{ + _oldposoccfile.reset(makePosOccRead(name, + _dictFile.get(), + dynamicKPosOccFormat, + featureParams, + tuneFileRead)); vespalib::string cname = prefix + "dictionary"; - vespalib::string name = prefix + "posocc.dat.compressed"; if (!_dictFile->open(cname, tuneFileRead)) { LOG(error, "Could not open posocc count file %s for read", cname.c_str()); + return false; } // open posocc.dat @@ -151,6 +143,7 @@ FieldReader::lateOpen(const vespalib::string &prefix, LOG(error, "Could not open posocc file %s for read", name.c_str()); + return false; } _oldWordNum = noWordNum(); _wordNum = _oldWordNum; @@ -160,17 +153,6 @@ FieldReader::lateOpen(const vespalib::string &prefix, return true; } - -bool -FieldReader::open(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) -{ - if (!earlyOpen(prefix, tuneFileRead)) - return false; - return lateOpen(prefix, tuneFileRead); -} - - bool FieldReader::close() { @@ -234,26 +216,6 @@ FieldReaderEmpty::FieldReaderEmpty(const IndexIterator &index) bool -FieldReaderEmpty::earlyOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) -{ - (void) prefix; - (void) tuneFileRead; - return true; -} - - -bool -FieldReaderEmpty::lateOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) -{ - (void) prefix; - (void) tuneFileRead; - return true; -} - - -bool FieldReaderEmpty::open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead) { diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h index aebb2fbe53a..d55aa39d491 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h @@ -116,14 +116,6 @@ public: const DocIdMapping &docIdMapping); virtual bool - earlyOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead); - - virtual bool - lateOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead); - - virtual bool open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead); virtual bool @@ -159,14 +151,6 @@ public: FieldReaderEmpty(const IndexIterator &index); virtual bool - earlyOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) override; - - virtual bool - lateOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) override; - - virtual bool open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead) override; @@ -200,4 +184,3 @@ public: } // namespace diskindex } // namespace search - diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp index 8b0c60d0bd7..6b66568ef7a 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp @@ -32,14 +32,15 @@ FieldWriter::FieldWriter(uint32_t docIdLimit, FieldWriter::~FieldWriter() { } -void -FieldWriter::earlyOpen(const vespalib::string &prefix, - uint32_t minSkipDocs, - uint32_t minChunkDocs, - bool dynamicKPosOccFormat, - const Schema &schema, - const uint32_t indexId, - const TuneFileSeqWrite &tuneFileWrite) +bool +FieldWriter::open(const vespalib::string &prefix, + uint32_t minSkipDocs, + uint32_t minChunkDocs, + bool dynamicKPosOccFormat, + const Schema &schema, + const uint32_t indexId, + const TuneFileSeqWrite &tuneFileWrite, + const FileHeaderContext &fileHeaderContext) { _prefix = prefix; vespalib::string name = prefix + "posocc.dat.compressed"; @@ -62,7 +63,7 @@ FieldWriter::earlyOpen(const vespalib::string &prefix, params.set("minChunkDocs", minChunkDocs); } - _dictFile.reset(new PageDict4FileSeqWrite); + _dictFile = std::make_unique<PageDict4FileSeqWrite>(); _dictFile->setParams(countParams); _posoccfile.reset(diskindex::makePosOccWrite(name, @@ -73,15 +74,7 @@ FieldWriter::earlyOpen(const vespalib::string &prefix, schema, indexId, tuneFileWrite)); -} - - -bool -FieldWriter::lateOpen(const TuneFileSeqWrite &tuneFileWrite, - const FileHeaderContext &fileHeaderContext) -{ vespalib::string cname = _prefix + "dictionary"; - vespalib::string name = _prefix + "posocc.dat.compressed"; // Open output dictionary file if (!_dictFile->open(cname, tuneFileWrite, fileHeaderContext)) { diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h index 893572fbdad..8f879360ea5 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h @@ -71,12 +71,10 @@ public: uint64_t getSparseWordNum() const { return _wordNum; } - void earlyOpen(const vespalib::string &prefix, uint32_t minSkipDocs, uint32_t minChunkDocs, - bool dynamicKPosOccFormat, const Schema &schema, uint32_t indexId, - const TuneFileSeqWrite &tuneFileWrite); - - bool lateOpen(const TuneFileSeqWrite &tuneFileWrite, - const search::common::FileHeaderContext &fileHeaderContext); + bool open(const vespalib::string &prefix, uint32_t minSkipDocs, uint32_t minChunkDocs, + bool dynamicKPosOccFormat, const Schema &schema, uint32_t indexId, + const TuneFileSeqWrite &tuneFileWrite, + const search::common::FileHeaderContext &fileHeaderContext); bool close(); diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index 8d6b9d04b51..5dc5af68a38 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -314,15 +314,14 @@ Fusion::openFieldWriter(const SchemaUtil::IndexIterator &index, { vespalib::string dir = _outDir + "/" + index.getName(); - writer.earlyOpen(dir + "/", + if (!writer.open(dir + "/", 64, 262144, _dynamicKPosIndexFormat, index.getSchema(), index.getIndex(), - _tuneFileIndexing._write); - if (!writer.lateOpen(_tuneFileIndexing._write, - _fileHeaderContext)) { + _tuneFileIndexing._write, + _fileHeaderContext)) { LOG(error, "Could not open output posocc + dictionary in %s", dir.c_str()); abort(); diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp index 0fdc4b96202..171e862f064 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp @@ -292,11 +292,9 @@ FileHandle::open(const vespalib::stringref &dir, _fieldWriter = new FieldWriter(docIdLimit, numWordIds); - _fieldWriter->earlyOpen(dir + "/", 64, 262144u, false, - index.getSchema(), index.getIndex(), - tuneFileWrite); - - if (!_fieldWriter->lateOpen(tuneFileWrite, fileHeaderContext)) { + if (!_fieldWriter->open(dir + "/", 64, 262144u, false, + index.getSchema(), index.getIndex(), + tuneFileWrite, fileHeaderContext)) { LOG(error, "Could not open term writer %s for write (%s)", dir.c_str(), getLastErrorString().c_str()); abort(); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp index e0aba63e595..1fa518af28f 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp @@ -553,7 +553,7 @@ FakeWord::validate(search::queryeval::SearchIterator *iterator, bool verbose) co bool -FakeWord::validate(std::shared_ptr<FieldReader> &fieldReader, +FakeWord::validate(FieldReader &fieldReader, uint32_t wordNum, const fef::TermFieldMatchDataArray &matchData, bool verbose) const @@ -582,8 +582,8 @@ FakeWord::validate(std::shared_ptr<FieldReader> &fieldReader, #endif numDocs = _postings.size(); for (residue = numDocs; residue > 0; --residue) { - assert(fieldReader->_wordNum == wordNum); - DocIdAndFeatures &features(fieldReader->_docIdAndFeatures); + assert(fieldReader._wordNum == wordNum); + DocIdAndFeatures &features(fieldReader._docIdAndFeatures); docId = features._docId; assert(d != de); assert(d->_docId == docId); @@ -645,7 +645,7 @@ FakeWord::validate(std::shared_ptr<FieldReader> &fieldReader, assert(presidue == 0); ++d; } - fieldReader->read(); + fieldReader.read(); } if (matchData.valid()) { assert(p == pe); @@ -699,7 +699,7 @@ FakeWord::validate(const search::BitVector &bv) const bool -FakeWord::dump(std::shared_ptr<FieldWriter> &fieldWriter, +FakeWord::dump(FieldWriter &fieldWriter, bool verbose) const { uint32_t numDocs; @@ -721,7 +721,7 @@ FakeWord::dump(std::shared_ptr<FieldWriter> &fieldWriter, assert(d != de); setupFeatures(*d, &*p, features); p += d->_positions; - fieldWriter->add(features); + fieldWriter.add(features); ++d; } assert(p == pe); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h index f65442d57e9..619c1760797 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h @@ -245,7 +245,7 @@ public: bool validate(search::queryeval::SearchIterator *iterator, bool verbose) const; bool - validate(std::shared_ptr<search::diskindex::FieldReader> &fieldReader, + validate(search::diskindex::FieldReader &fieldReader, uint32_t wordNum, const fef::TermFieldMatchDataArray &matchData, bool verbose) const; @@ -254,7 +254,7 @@ public: void validate(const BitVector &bv) const; bool - dump(std::shared_ptr<search::diskindex::FieldWriter> &fieldWriter, + dump(search::diskindex::FieldWriter &fieldWriter, bool verbose) const; const std::string &getName() const { return _name; } |