summaryrefslogtreecommitdiffstats
path: root/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp')
-rw-r--r--searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp250
1 files changed, 17 insertions, 233 deletions
diff --git a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp
index 72a50d87821..71467519dbd 100644
--- a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp
+++ b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp
@@ -9,10 +9,8 @@
#include <vespa/searchlib/index/postinglisthandle.h>
#include <vespa/searchlib/diskindex/zcposocc.h>
#include <vespa/searchlib/diskindex/zcposoccrandread.h>
-#include <vespa/searchlib/diskindex/checkpointfile.h>
#include <vespa/searchlib/index/dummyfileheadercontext.h>
#include <vespa/searchlib/index/schemautil.h>
-#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/searchlib/diskindex/fieldwriter.h>
#include <vespa/searchlib/diskindex/fieldreader.h>
#include <vespa/vespalib/io/fileutil.h>
@@ -29,7 +27,6 @@ using search::TuneFileRandRead;
using search::TuneFileSeqRead;
using search::TuneFileSeqWrite;
using search::common::FileHeaderContext;
-using search::diskindex::CheckPointFile;
using search::diskindex::DocIdMapping;
using search::diskindex::FieldReader;
using search::diskindex::FieldWriter;
@@ -48,7 +45,6 @@ using search::index::SchemaUtil;
using search::index::schema::CollectionType;
using search::index::schema::DataType;
using search::queryeval::SearchIterator;
-using vespalib::nbostream;
using namespace search::index;
@@ -91,9 +87,6 @@ makeWordString(uint64_t wordNum)
}
-typedef std::shared_ptr<FieldReader> FieldReaderSP;
-typedef std::shared_ptr<FieldWriter> FieldWriterSP;
-
class FieldWriterTest : public FastOS_Application
{
private:
@@ -144,10 +137,10 @@ FieldWriterTest::~FieldWriterTest()
}
-class WrappedFieldWriter : public search::fakedata::CheckPointCallback
+class WrappedFieldWriter
{
public:
- FieldWriterSP _fieldWriter;
+ std::unique_ptr<FieldWriter> _fieldWriter;
private:
bool _dynamicK;
uint32_t _numWordIds;
@@ -164,13 +157,8 @@ public:
uint32_t docIdLimit);
~WrappedFieldWriter();
- void checkPoint() override;
- void earlyOpen();
- void lateOpen();
void open();
void close();
- void writeCheckPoint();
- void readCheckPoint(bool first);
};
WrappedFieldWriter::~WrappedFieldWriter() {}
@@ -194,32 +182,16 @@ WrappedFieldWriter::WrappedFieldWriter(const vespalib::string &namepref,
void
-WrappedFieldWriter::earlyOpen()
-{
- TuneFileSeqWrite tuneFileWrite;
- _fieldWriter.reset(new FieldWriter(_docIdLimit, _numWordIds));
- _fieldWriter->earlyOpen(_namepref,
- minSkipDocs, minChunkDocs, _dynamicK, _schema,
- _indexId,
- tuneFileWrite);
-}
-
-
-void
-WrappedFieldWriter::lateOpen()
+WrappedFieldWriter::open()
{
TuneFileSeqWrite tuneFileWrite;
DummyFileHeaderContext fileHeaderContext;
fileHeaderContext.disableFileName();
- _fieldWriter->lateOpen(tuneFileWrite, fileHeaderContext);
-}
-
-
-void
-WrappedFieldWriter::open()
-{
- earlyOpen();
- lateOpen();
+ _fieldWriter = std::make_unique<FieldWriter>(_docIdLimit, _numWordIds);
+ _fieldWriter->open(_namepref,
+ minSkipDocs, minChunkDocs, _dynamicK, _schema,
+ _indexId,
+ tuneFileWrite, fileHeaderContext);
}
@@ -231,46 +203,10 @@ WrappedFieldWriter::close()
}
-void
-WrappedFieldWriter::writeCheckPoint()
-{
- CheckPointFile chkptfile("chkpt");
- nbostream out;
- _fieldWriter->checkPointWrite(out);
- chkptfile.write(out, DummyFileHeaderContext());
-}
-
-
-void
-WrappedFieldWriter::readCheckPoint(bool first)
-{
- CheckPointFile chkptfile("chkpt");
- nbostream in;
- bool openRes = chkptfile.read(in);
- assert(first || openRes);
- (void) first;
- if (!openRes)
- return;
- _fieldWriter->checkPointRead(in);
- assert(in.empty());
-}
-
-
-void
-WrappedFieldWriter::checkPoint()
-{
- writeCheckPoint();
- _fieldWriter.reset();
- earlyOpen();
- readCheckPoint(false);
- lateOpen();
-}
-
-
-class WrappedFieldReader : public search::fakedata::CheckPointCallback
+class WrappedFieldReader
{
public:
- FieldReaderSP _fieldReader;
+ std::unique_ptr<FieldReader> _fieldReader;
private:
std::string _namepref;
uint32_t _numWordIds;
@@ -286,21 +222,15 @@ public:
uint32_t docIdLimit);
~WrappedFieldReader();
- void earlyOpen();
- void lateOpen();
void open();
void close();
- void writeCheckPoint();
- void readCheckPoint(bool first);
- virtual void checkPoint() override;
};
WrappedFieldReader::WrappedFieldReader(const vespalib::string &namepref,
uint32_t numWordIds,
uint32_t docIdLimit)
- : search::fakedata::CheckPointCallback(),
- _fieldReader(),
+ : _fieldReader(),
_namepref(dirprefix + namepref),
_numWordIds(numWordIds),
_docIdLimit(docIdLimit),
@@ -323,35 +253,17 @@ WrappedFieldReader::~WrappedFieldReader()
{
}
-
-void
-WrappedFieldReader::earlyOpen()
-{
- TuneFileSeqRead tuneFileRead;
- _fieldReader.reset(new FieldReader());
- _fieldReader->earlyOpen(_namepref, tuneFileRead);
-}
-
-
void
-WrappedFieldReader::lateOpen()
+WrappedFieldReader::open()
{
TuneFileSeqRead tuneFileRead;
_wmap.setup(_numWordIds);
_dmap.setup(_docIdLimit);
+ _fieldReader = std::make_unique<FieldReader>();
_fieldReader->setup(_wmap, _dmap);
- _fieldReader->lateOpen(_namepref, tuneFileRead);
-}
-
-
-void
-WrappedFieldReader::open()
-{
- earlyOpen();
- lateOpen();
+ _fieldReader->open(_namepref, tuneFileRead);
}
-
void
WrappedFieldReader::close()
{
@@ -361,42 +273,6 @@ WrappedFieldReader::close()
void
-WrappedFieldReader::writeCheckPoint()
-{
- CheckPointFile chkptfile("chkpt");
- nbostream out;
- _fieldReader->checkPointWrite(out);
- chkptfile.write(out, DummyFileHeaderContext());
-}
-
-
-void
-WrappedFieldReader::readCheckPoint(bool first)
-{
- CheckPointFile chkptfile("chkpt");
- nbostream in;
- bool openRes = chkptfile.read(in);
- assert(first || openRes);
- (void) first;
- if (!openRes)
- return;
- _fieldReader->checkPointRead(in);
- assert(in.empty());
-}
-
-
-void
-WrappedFieldReader::checkPoint()
-{
- writeCheckPoint();
- _fieldReader.reset();
- earlyOpen();
- readCheckPoint(false);
- lateOpen();
-}
-
-
-void
writeField(FakeWordSet &wordSet,
uint32_t docIdLimit,
const std::string &namepref,
@@ -422,16 +298,11 @@ writeField(FakeWordSet &wordSet,
ostate.open();
unsigned int wordNum = 1;
- uint32_t checkPointCheck = 0;
- uint32_t checkPointInterval = 12227;
for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) {
for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) {
FakeWord &fw = *wordSet._words[wc][wi];
ostate._fieldWriter->newWord(makeWordString(wordNum));
- fw.dump(ostate._fieldWriter, false,
- checkPointCheck,
- checkPointInterval,
- NULL);
+ fw.dump(*ostate._fieldWriter, false);
++wordNum;
}
}
@@ -450,74 +321,6 @@ writeField(FakeWordSet &wordSet,
void
-writeFieldCheckPointed(FakeWordSet &wordSet,
- uint32_t docIdLimit,
- const std::string &namepref,
- bool dynamicK)
-{
- const char *dynamicKStr = dynamicK ? "true" : "false";
-
- FastOS_Time tv;
- double before;
- double after;
- bool first = true;
-
- LOG(info,
- "enter writeFieldCheckPointed, "
- "namepref=%s, dynamicK=%s",
- namepref.c_str(),
- dynamicKStr);
- tv.SetNow();
- before = tv.Secs();
-
- unsigned int wordNum = 1;
- uint32_t checkPointCheck = 0;
- uint32_t checkPointInterval = 12227;
- for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) {
- for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) {
- FakeWord &fw = *wordSet._words[wc][wi];
-
- WrappedFieldWriter ostate(namepref,
- dynamicK,
- wordSet.getNumWords(), docIdLimit);
- ostate.earlyOpen();
- ostate.readCheckPoint(first);
- first = false;
- ostate.lateOpen();
- ostate._fieldWriter->newWord(makeWordString(wordNum));
- fw.dump(ostate._fieldWriter, false,
- checkPointCheck,
- checkPointInterval,
- &ostate);
- ostate.writeCheckPoint();
- ++wordNum;
- }
- }
- do {
- WrappedFieldWriter ostate(namepref,
- dynamicK,
- wordSet.getNumWords(), docIdLimit);
- ostate.earlyOpen();
- ostate.readCheckPoint(first);
- ostate.lateOpen();
- ostate.close();
- } while (0);
- CheckPointFile dropper("chkpt");
- dropper.remove();
-
- tv.SetNow();
- after = tv.Secs();
- LOG(info,
- "leave writeFieldCheckPointed, "
- "namepref=%s, dynamicK=%s"
- " elapsed=%10.6f",
- namepref.c_str(),
- dynamicKStr,
- after - before);
-}
-
-
-void
readField(FakeWordSet &wordSet,
uint32_t docIdLimit,
const std::string &namepref,
@@ -545,8 +348,6 @@ readField(FakeWordSet &wordSet,
TermFieldMatchData mdfield1;
unsigned int wordNum = 1;
- uint32_t checkPointCheck = 0;
- uint32_t checkPointInterval = 12227;
for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) {
for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) {
FakeWord &fw = *wordSet._words[wc][wi];
@@ -554,9 +355,8 @@ readField(FakeWordSet &wordSet,
TermFieldMatchDataArray tfmda;
tfmda.add(&mdfield1);
- fw.validate(istate._fieldReader, wordNum,
- tfmda, verbose,
- checkPointCheck, checkPointInterval, &istate);
+ fw.validate(*istate._fieldReader, wordNum,
+ tfmda, verbose);
++wordNum;
}
}
@@ -564,8 +364,6 @@ readField(FakeWordSet &wordSet,
istate.close();
tv.SetNow();
after = tv.Secs();
- CheckPointFile dropper("chkpt");
- dropper.remove();
LOG(info,
"leave readField, "
"namepref=%s, dynamicK=%s"
@@ -762,35 +560,23 @@ void
testFieldWriterVariants(FakeWordSet &wordSet,
uint32_t docIdLimit, bool verbose)
{
- CheckPointFile dropper("chkpt");
- dropper.remove();
disableSkip();
writeField(wordSet, docIdLimit, "new4", true);
readField(wordSet, docIdLimit, "new4", true, verbose);
readField(wordSet, docIdLimit, "new4", true, verbose);
- writeFieldCheckPointed(wordSet, docIdLimit, "new6", true);
writeField(wordSet, docIdLimit, "new5", false);
readField(wordSet, docIdLimit, "new5", false, verbose);
- writeFieldCheckPointed(wordSet, docIdLimit, "new7", false);
enableSkip();
writeField(wordSet, docIdLimit, "newskip4", true);
readField(wordSet, docIdLimit, "newskip4", true, verbose);
- writeFieldCheckPointed(wordSet, docIdLimit, "newskip6",
- true);
writeField(wordSet, docIdLimit, "newskip5", false);
readField(wordSet, docIdLimit, "newskip5", false, verbose);
- writeFieldCheckPointed(wordSet, docIdLimit, "newskip7",
- false);
enableSkipChunks();
writeField(wordSet, docIdLimit, "newchunk4", true);
readField(wordSet, docIdLimit, "newchunk4", true, verbose);
- writeFieldCheckPointed(wordSet, docIdLimit, "newchunk6",
- true);
writeField(wordSet, docIdLimit, "newchunk5", false);
readField(wordSet, docIdLimit,
"newchunk5",false, verbose);
- writeFieldCheckPointed(wordSet, docIdLimit, "newchunk7",
- false);
disableSkip();
fusionField(wordSet.getNumWords(),
docIdLimit,
@@ -855,8 +641,6 @@ void
testFieldWriterVariantsWithHighLids(FakeWordSet &wordSet, uint32_t docIdLimit,
bool verbose)
{
- CheckPointFile dropper("chkpt");
- dropper.remove();
disableSkip();
writeField(wordSet, docIdLimit, "hlid4", true);
readField(wordSet, docIdLimit, "hlid4", true, verbose);