Stop testing disk index checkpointing. It is not currently used.

6 files changed, 9 insertions, 446 deletions

diff --git a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp
index 72a50d87821..0746f62b1df 100644
--- a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp
+++ b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp
@@ -9,10 +9,8 @@
 #include <vespa/searchlib/index/postinglisthandle.h>
 #include <vespa/searchlib/diskindex/zcposocc.h>
 #include <vespa/searchlib/diskindex/zcposoccrandread.h>
-#include <vespa/searchlib/diskindex/checkpointfile.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/index/schemautil.h>
-#include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/searchlib/diskindex/fieldwriter.h>
 #include <vespa/searchlib/diskindex/fieldreader.h>
 #include <vespa/vespalib/io/fileutil.h>
@@ -29,7 +27,6 @@ using search::TuneFileRandRead;
 using search::TuneFileSeqRead;
 using search::TuneFileSeqWrite;
 using search::common::FileHeaderContext;
-using search::diskindex::CheckPointFile;
 using search::diskindex::DocIdMapping;
 using search::diskindex::FieldReader;
 using search::diskindex::FieldWriter;
@@ -48,7 +45,6 @@ using search::index::SchemaUtil;
 using search::index::schema::CollectionType;
 using search::index::schema::DataType;
 using search::queryeval::SearchIterator;
-using vespalib::nbostream;
 
 using namespace search::index;
 
@@ -144,7 +140,7 @@ FieldWriterTest::~FieldWriterTest()
 }
 
 
-class WrappedFieldWriter : public search::fakedata::CheckPointCallback
+class WrappedFieldWriter
 {
 public:
     FieldWriterSP _fieldWriter;
@@ -164,13 +160,10 @@ public:
                       uint32_t docIdLimit);
     ~WrappedFieldWriter();
 
-    void checkPoint() override;
     void earlyOpen();
     void lateOpen();
     void open();
     void close();
-    void writeCheckPoint();
-    void readCheckPoint(bool first);
 };
 
 WrappedFieldWriter::~WrappedFieldWriter() {}
@@ -231,43 +224,7 @@ WrappedFieldWriter::close()
 }
 
 
-void
-WrappedFieldWriter::writeCheckPoint()
-{
-    CheckPointFile chkptfile("chkpt");
-    nbostream out;
-    _fieldWriter->checkPointWrite(out);
-    chkptfile.write(out, DummyFileHeaderContext());
-}
-
-
-void
-WrappedFieldWriter::readCheckPoint(bool first)
-{
-    CheckPointFile chkptfile("chkpt");
-    nbostream in;
-    bool openRes = chkptfile.read(in);
-    assert(first || openRes);
-    (void) first;
-    if (!openRes)
-        return;
-    _fieldWriter->checkPointRead(in);
-    assert(in.empty());
-}
-
-
-void
-WrappedFieldWriter::checkPoint()
-{
-    writeCheckPoint();
-    _fieldWriter.reset();
-    earlyOpen();
-    readCheckPoint(false);
-    lateOpen();
-}
-
-
-class WrappedFieldReader : public search::fakedata::CheckPointCallback
+class WrappedFieldReader
 {
 public:
     FieldReaderSP _fieldReader;
@@ -290,17 +247,13 @@ public:
     void lateOpen();
     void open();
     void close();
-    void writeCheckPoint();
-    void readCheckPoint(bool first);
-    virtual void checkPoint() override;
 };
 
 
 WrappedFieldReader::WrappedFieldReader(const vespalib::string &namepref,
                                      uint32_t numWordIds,
                                      uint32_t docIdLimit)
-    : search::fakedata::CheckPointCallback(),
-      _fieldReader(),
+    : _fieldReader(),
       _namepref(dirprefix + namepref),
       _numWordIds(numWordIds),
       _docIdLimit(docIdLimit),
@@ -361,42 +314,6 @@ WrappedFieldReader::close()
 
 
 void
-WrappedFieldReader::writeCheckPoint()
-{
-    CheckPointFile chkptfile("chkpt");
-    nbostream out;
-    _fieldReader->checkPointWrite(out);
-    chkptfile.write(out, DummyFileHeaderContext());
-}
-
-
-void
-WrappedFieldReader::readCheckPoint(bool first)
-{
-    CheckPointFile chkptfile("chkpt");
-    nbostream in;
-    bool openRes = chkptfile.read(in);
-    assert(first || openRes);
-    (void) first;
-    if (!openRes)
-        return;
-    _fieldReader->checkPointRead(in);
-    assert(in.empty());
-}
-
-
-void
-WrappedFieldReader::checkPoint()
-{
-    writeCheckPoint();
-    _fieldReader.reset();
-    earlyOpen();
-    readCheckPoint(false);
-    lateOpen();
-}
-
-
-void
 writeField(FakeWordSet &wordSet,
            uint32_t docIdLimit,
            const std::string &namepref,
@@ -422,16 +339,11 @@ writeField(FakeWordSet &wordSet,
     ostate.open();
 
     unsigned int wordNum = 1;
-    uint32_t checkPointCheck = 0;
-    uint32_t checkPointInterval = 12227;
     for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) {
         for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) {
             FakeWord &fw = *wordSet._words[wc][wi];
             ostate._fieldWriter->newWord(makeWordString(wordNum));
-            fw.dump(ostate._fieldWriter, false,
-                    checkPointCheck,
-                    checkPointInterval,
-                    NULL);
+            fw.dump(ostate._fieldWriter, false);
             ++wordNum;
         }
     }
@@ -450,74 +362,6 @@ writeField(FakeWordSet &wordSet,
 
 
 void
-writeFieldCheckPointed(FakeWordSet &wordSet,
-             uint32_t docIdLimit,
-             const std::string &namepref,
-             bool dynamicK)
-{
-    const char *dynamicKStr = dynamicK ? "true" : "false";
-
-    FastOS_Time tv;
-    double before;
-    double after;
-    bool first = true;
-
-    LOG(info,
-        "enter writeFieldCheckPointed, "
-        "namepref=%s, dynamicK=%s",
-        namepref.c_str(),
-        dynamicKStr);
-    tv.SetNow();
-    before = tv.Secs();
-
-    unsigned int wordNum = 1;
-    uint32_t checkPointCheck = 0;
-    uint32_t checkPointInterval = 12227;
-    for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) {
-        for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) {
-            FakeWord &fw = *wordSet._words[wc][wi];
-
-            WrappedFieldWriter ostate(namepref,
-                                     dynamicK,
-                                     wordSet.getNumWords(), docIdLimit);
-            ostate.earlyOpen();
-            ostate.readCheckPoint(first);
-            first = false;
-            ostate.lateOpen();
-            ostate._fieldWriter->newWord(makeWordString(wordNum));
-            fw.dump(ostate._fieldWriter, false,
-                    checkPointCheck,
-                    checkPointInterval,
-                    &ostate);
-            ostate.writeCheckPoint();
-            ++wordNum;
-        }
-    }
-    do {
-        WrappedFieldWriter ostate(namepref,
-                                 dynamicK,
-                                 wordSet.getNumWords(), docIdLimit);
-        ostate.earlyOpen();
-        ostate.readCheckPoint(first);
-        ostate.lateOpen();
-        ostate.close();
-    } while (0);
-    CheckPointFile dropper("chkpt");
-    dropper.remove();
-
-    tv.SetNow();
-    after = tv.Secs();
-    LOG(info,
-        "leave writeFieldCheckPointed, "
-        "namepref=%s, dynamicK=%s"
-        " elapsed=%10.6f",
-        namepref.c_str(),
-        dynamicKStr,
-        after - before);
-}
-
-
-void
 readField(FakeWordSet &wordSet,
           uint32_t docIdLimit,
           const std::string &namepref,
@@ -545,8 +389,6 @@ readField(FakeWordSet &wordSet,
     TermFieldMatchData mdfield1;
 
     unsigned int wordNum = 1;
-    uint32_t checkPointCheck = 0;
-    uint32_t checkPointInterval = 12227;
     for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) {
         for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) {
             FakeWord &fw = *wordSet._words[wc][wi];
@@ -555,8 +397,7 @@ readField(FakeWordSet &wordSet,
             tfmda.add(&mdfield1);
 
             fw.validate(istate._fieldReader, wordNum,
-                        tfmda, verbose,
-                        checkPointCheck, checkPointInterval, &istate);
+                        tfmda, verbose);
             ++wordNum;
         }
     }
@@ -564,8 +405,6 @@ readField(FakeWordSet &wordSet,
     istate.close();
     tv.SetNow();
     after = tv.Secs();
-    CheckPointFile dropper("chkpt");
-    dropper.remove();
     LOG(info,
         "leave readField, "
         "namepref=%s, dynamicK=%s"
@@ -762,35 +601,23 @@ void
 testFieldWriterVariants(FakeWordSet &wordSet,
                         uint32_t docIdLimit, bool verbose)
 {
-    CheckPointFile dropper("chkpt");
-    dropper.remove();
     disableSkip();
     writeField(wordSet, docIdLimit, "new4", true);
     readField(wordSet, docIdLimit, "new4", true, verbose);
     readField(wordSet, docIdLimit, "new4", true, verbose);
-    writeFieldCheckPointed(wordSet, docIdLimit, "new6", true);
     writeField(wordSet, docIdLimit, "new5", false);
     readField(wordSet, docIdLimit, "new5", false, verbose);
-    writeFieldCheckPointed(wordSet, docIdLimit, "new7", false);
     enableSkip();
     writeField(wordSet, docIdLimit, "newskip4", true);
     readField(wordSet, docIdLimit, "newskip4", true, verbose);
-    writeFieldCheckPointed(wordSet, docIdLimit, "newskip6",
-                                      true);
     writeField(wordSet, docIdLimit, "newskip5", false);
     readField(wordSet, docIdLimit, "newskip5", false, verbose);
-    writeFieldCheckPointed(wordSet, docIdLimit, "newskip7",
-                                      false);
     enableSkipChunks();
     writeField(wordSet, docIdLimit, "newchunk4", true);
     readField(wordSet, docIdLimit, "newchunk4", true, verbose);
-    writeFieldCheckPointed(wordSet, docIdLimit, "newchunk6",
-                                      true);
     writeField(wordSet, docIdLimit, "newchunk5", false);
     readField(wordSet, docIdLimit,
                 "newchunk5",false, verbose);
-    writeFieldCheckPointed(wordSet, docIdLimit, "newchunk7",
-                                      false);
     disableSkip();
     fusionField(wordSet.getNumWords(),
                 docIdLimit,
@@ -855,8 +682,6 @@ void
 testFieldWriterVariantsWithHighLids(FakeWordSet &wordSet, uint32_t docIdLimit,
                              bool verbose)
 {
-    CheckPointFile dropper("chkpt");
-    dropper.remove();
     disableSkip();
     writeField(wordSet, docIdLimit, "hlid4", true);
     readField(wordSet, docIdLimit, "hlid4", true, verbose);
diff --git a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt
index 9b6bf497c9a..3619affb54e 100644
--- a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt
@@ -5,7 +5,6 @@ vespa_add_library(searchlib_diskindex OBJECT
     bitvectorfile.cpp
     bitvectoridxfile.cpp
     bitvectorkeyscope.cpp
-    checkpointfile.cpp
     dictionarywordreader.cpp
     diskindex.cpp
     disktermblueprint.cpp
diff --git a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp b/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp
deleted file mode 100644
index 0324f00f63c..00000000000
--- a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp
+++ /dev/null
@@ -1,183 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "checkpointfile.h"
-#include <vespa/vespalib/data/fileheader.h>
-#include <vespa/searchlib/common/fileheadercontext.h>
-#include <cassert>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".diskindex.checkpointfile");
-
-using vespalib::getLastErrorString;
-
-namespace search::diskindex {
-
-using common::FileHeaderContext;
-
-CheckPointFile::CheckPointFile(const vespalib::string &name)
-    : _file(),
-      _name(name),
-      _nameNew(name + ".NEW"),
-      _nameNewNew(name + ".NEW.NEW"),
-      _writeOpened(false),
-      _headerLen(0u)
-{ }
-
-
-CheckPointFile::~CheckPointFile()
-{
-    close();
-}
-
-
-void
-CheckPointFile::writeOpen(const FileHeaderContext &fileHeaderContext)
-{
-    FastOS_File::Delete(_nameNewNew.c_str());
-    _file.OpenWriteOnly(_nameNewNew.c_str());
-    _writeOpened = true;
-    makeHeader(fileHeaderContext);
-}
-
-
-bool
-CheckPointFile::readOpen()
-{
-    bool openres;
-
-    openres = _file.OpenReadOnly(_name.c_str());
-    if (!openres) {
-        bool renameres = FastOS_File::Rename(_nameNew.c_str(),
-                _name.c_str());
-        if (!renameres)
-            return false;
-        openres = _file.OpenReadOnly(_name.c_str());
-        if (!openres)
-            return false;
-    }
-    _headerLen = readHeader();
-    return true;
-}
-
-
-void
-CheckPointFile::close()
-{
-    if (_writeOpened) {
-        _file.Sync();
-    }
-    _file.Close();
-    if (_writeOpened) {
-        updateHeader();
-        rename1();
-        rename2();
-    }
-    _writeOpened = false;
-}
-
-
-void
-CheckPointFile::rename1()
-{
-    FastOS_File::Delete(_nameNew.c_str());
-    bool renameres = FastOS_File::Rename(_nameNewNew.c_str(),
-            _nameNew.c_str());
-    if (!renameres) {
-        LOG(error, "FATAL: rename %s -> %s failed: %s",
-            _nameNewNew.c_str(), _nameNew.c_str(), getLastErrorString().c_str());
-        abort();
-    }
-}
-
-
-void
-CheckPointFile::rename2()
-{
-    FastOS_File::Delete(_name.c_str());
-    bool renameres = FastOS_File::Rename(_nameNew.c_str(), _name.c_str());
-    if (!renameres) {
-        LOG(error, "FATAL: rename %s -> %s failed: %s",
-            _nameNew.c_str(), _name.c_str(), getLastErrorString().c_str());
-        abort();
-    }
-}
-
-
-void
-CheckPointFile::remove()
-{
-    FastOS_File::Delete(_nameNew.c_str());
-    FastOS_File::Delete(_name.c_str());
-}
-
-
-
-void
-CheckPointFile::write(vespalib::nbostream &buf,
-                      const FileHeaderContext &fileHeaderContext)
-{
-    writeOpen(fileHeaderContext);
-    _file.WriteBuf(buf.peek(), buf.size());
-    close();
-}
-
-
-bool
-CheckPointFile::read(vespalib::nbostream &buf)
-{
-    if (!readOpen())
-        return false;
-    size_t sz = _file.GetSize() - _headerLen;
-
-    std::vector<char> tmp(sz);
-    _file.ReadBuf(&tmp[0], sz);
-    buf.clear();
-    buf.write(&tmp[0], sz);
-    std::vector<char>().swap(tmp);
-    close();
-    return true;
-}
-
-
-void
-CheckPointFile::makeHeader(const FileHeaderContext &fileHeaderContext)
-{
-    vespalib::FileHeader header;
-
-    typedef vespalib::GenericHeader::Tag Tag;
-    fileHeaderContext.addTags(header, _file.GetFileName());
-    header.putTag(Tag("frozen", 0));
-    header.putTag(Tag("desc", "Check point file"));
-    header.writeFile(_file);
-}
-
-
-void
-CheckPointFile::updateHeader()
-{
-    vespalib::FileHeader h;
-    FastOS_File f;
-    f.OpenReadWrite(_nameNewNew.c_str());
-    h.readFile(f);
-    FileHeaderContext::setFreezeTime(h);
-    typedef vespalib::GenericHeader::Tag Tag;
-    h.putTag(Tag("frozen", 1));
-    h.rewriteFile(f);
-    f.Sync();
-    f.Close();
-}
-
-
-uint32_t
-CheckPointFile::readHeader()
-{
-    vespalib::FileHeader h;
-    uint32_t headerLen = h.readFile(_file);
-    _file.SetPosition(headerLen);
-    assert(h.hasTag("frozen"));
-    assert(h.getTag("frozen").asInteger() != 0);
-    return headerLen;
-}
-
-
-}
diff --git a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.h b/searchlib/src/vespa/searchlib/diskindex/checkpointfile.h
deleted file mode 100644
index 7ea18132b08..00000000000
--- a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.h
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/objects/nbostream.h>
-#include <vespa/fastos/file.h>
-
-namespace search {
-
-namespace common { class FileHeaderContext; }
-
-namespace diskindex {
-
-class CheckPointFile
-{
-public:
-    FastOS_File _file;
-    vespalib::string _name;
-    vespalib::string _nameNew;
-    vespalib::string _nameNewNew;
-    bool _writeOpened;
-    uint32_t _headerLen;
-
-    void writeOpen(const common::FileHeaderContext &fileHeaderContext);
-    bool readOpen();
-    void close();
-    void rename1();
-    void rename2();
-    void remove();
-    void makeHeader(const common::FileHeaderContext &fileHeaderContext);
-    void updateHeader();
-    uint32_t readHeader();
-public:
-    CheckPointFile(const CheckPointFile &) = delete;
-    CheckPointFile & operator = (const CheckPointFile &) = delete;
-    CheckPointFile(const vespalib::string &name);
-    ~CheckPointFile();
-
-    void write(vespalib::nbostream &buf, const common::FileHeaderContext &fileHeaderContext);
-    bool read(vespalib::nbostream &buf);
-};
-
-
-} // namespace diskindex
-
-} // namespace search
-
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp
index bfc7f7032c1..e0aba63e595 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp
@@ -556,10 +556,7 @@ bool
 FakeWord::validate(std::shared_ptr<FieldReader> &fieldReader,
                    uint32_t wordNum,
                    const fef::TermFieldMatchDataArray &matchData,
-                   bool verbose,
-                   uint32_t &checkPointCheck,
-                   uint32_t checkPointInterval,
-                   CheckPointCallback *const checkPointCallback) const
+                   bool verbose) const
 {
     uint32_t docId = 0;
     uint32_t numDocs;
@@ -648,11 +645,6 @@ FakeWord::validate(std::shared_ptr<FieldReader> &fieldReader,
             assert(presidue == 0);
             ++d;
         }
-        if (++checkPointCheck >= checkPointInterval) {
-            checkPointCheck = 0;
-            if (checkPointCallback != NULL)
-                checkPointCallback->checkPoint();
-        }
         fieldReader->read();
     }
     if (matchData.valid()) {
@@ -708,10 +700,7 @@ FakeWord::validate(const search::BitVector &bv) const
 
 bool
 FakeWord::dump(std::shared_ptr<FieldWriter> &fieldWriter,
-               bool verbose,
-               uint32_t &checkPointCheck,
-               uint32_t checkPointInterval,
-               CheckPointCallback *checkPointCallback) const
+               bool verbose) const
 {
     uint32_t numDocs;
     uint32_t residue;
@@ -734,11 +723,6 @@ FakeWord::dump(std::shared_ptr<FieldWriter> &fieldWriter,
         p += d->_positions;
         fieldWriter->add(features);
         ++d;
-        if (++checkPointCheck >= checkPointInterval) {
-            checkPointCheck = 0;
-            if (checkPointCallback != NULL)
-                checkPointCallback->checkPoint();
-        }
     }
     assert(p == pe);
     assert(d == de);
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h
index 9417c32f1e5..f65442d57e9 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h
@@ -17,15 +17,6 @@ namespace search {
 namespace fakedata {
 
 
-class CheckPointCallback
-{
-public:
-    CheckPointCallback() {}
-    virtual ~CheckPointCallback() {}
-
-    virtual void checkPoint() = 0;
-};
-
 /*
  * General representation of a faked word, containing all features used
  * by any of the candidate posting list formats.
@@ -257,20 +248,14 @@ public:
     validate(std::shared_ptr<search::diskindex::FieldReader> &fieldReader,
              uint32_t wordNum,
              const fef::TermFieldMatchDataArray &matchData,
-             bool verbose,
-             uint32_t &checkPointCheck,
-             uint32_t checkPointInterval,
-             CheckPointCallback *const checkPointCallback) const;
+             bool verbose) const;
 
     void validate(const std::vector<uint32_t> &docIds) const;
     void validate(const BitVector &bv) const;
 
     bool
     dump(std::shared_ptr<search::diskindex::FieldWriter> &fieldWriter,
-         bool verbose,
-         uint32_t &checkPointCheck,
-         uint32_t checkPointInterval,
-         CheckPointCallback *checkPointCallback) const;
+         bool verbose) const;
 
     const std::string &getName() const { return _name; }
     uint32_t getDocIdLimit() const { return _docIdLimit; }