diff options
56 files changed, 127 insertions, 2183 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index a1fd3ffd729..cc78c2715e2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -13,7 +13,6 @@ import com.yahoo.yolean.Exceptions; import java.io.PrintWriter; import java.io.StringWriter; -import java.time.Duration; import java.util.concurrent.Executor; /** @@ -70,13 +69,6 @@ public class HttpHandler extends LoggingRequestHandler { } } - // Override default, since we need a higher timeout for some calls - // TODO: Review and see if overriding only in SessionPrepareHandler is enough - @Override - public Duration getTimeout() { - return Duration.ofSeconds(910); - } - private String getMessage(Exception e, HttpRequest request) { if (request.getBooleanProperty("debug")) { StringWriter sw = new StringWriter(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index 93796294cd3..2947ef68ba4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * The responsibility of this class is to configure ACLs for all running containers. The ACLs are fetched from the Node @@ -63,15 +64,13 @@ public class AclMaintainer implements Runnable { } final Command flush = new FlushCommand(Chain.INPUT); final Command rollback = new PolicyCommand(Chain.INPUT, Action.ACCEPT); - log.info("Start modifying ACL rules for " + containerName.asString()); try { - log.debug("Running ACL command '" + flush.asString() + "'"); - dockerOperations.executeCommandInNetworkNamespace(containerName, flush.asArray(IPTABLES_COMMAND)); - acl.toCommands().forEach(command -> { - log.debug("Running ACL command '" + command.asString() + "' for " + containerName.asString()); - dockerOperations.executeCommandInNetworkNamespace(containerName, - command.asArray(IPTABLES_COMMAND)); - }); + String commands = Stream.concat(Stream.of(flush), acl.toCommands().stream()) + .map(command -> command.asString(IPTABLES_COMMAND)) + .collect(Collectors.joining("; ")); + + log.debug("Running ACL command '" + commands + "' in " + containerName.asString()); + dockerOperations.executeCommandInNetworkNamespace(containerName, "/bin/sh", "-c", commands); containerAcls.put(containerName, acl); } catch (Exception e) { log.error("Exception occurred while configuring ACLs for " + containerName.asString() + ", attempting rollback", e); @@ -81,7 +80,6 @@ public class AclMaintainer implements Runnable { log.error("Rollback of ACLs for " + containerName.asString() + " failed, giving up", ne); } } - log.info("Finished modifying ACL rules for " + containerName.asString()); } private synchronized void configureAcls() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java index 17e5637c0eb..9ce48dac55b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -116,83 +116,23 @@ public class AclMaintainerTest { private void assertAclsApplied(ContainerName containerName, List<ContainerAclSpec> containerAclSpecs, VerificationMode verificationMode) { + StringBuilder expectedCommand = new StringBuilder() + .append("ip6tables -F INPUT; ") + .append("ip6tables -P INPUT DROP; ") + .append("ip6tables -P FORWARD DROP; ") + .append("ip6tables -P OUTPUT ACCEPT; ") + .append("ip6tables -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT; ") + .append("ip6tables -A INPUT -i lo -j ACCEPT; ") + .append("ip6tables -A INPUT -p ipv6-icmp -j ACCEPT; "); + + containerAclSpecs.forEach(aclSpec -> + expectedCommand.append("ip6tables -A INPUT -s " + aclSpec.ipAddress() + "/128 -j ACCEPT; ")); + + expectedCommand.append("ip6tables -A INPUT -j REJECT"); + + verify(dockerOperations, verificationMode).executeCommandInNetworkNamespace( - eq(containerName), - eq("ip6tables"), - eq("-F"), - eq("INPUT") - ); - verify(dockerOperations, verificationMode).executeCommandInNetworkNamespace( - eq(containerName), - eq("ip6tables"), - eq("-P"), - eq("INPUT"), - eq("DROP") - ); - verify(dockerOperations, verificationMode).executeCommandInNetworkNamespace( - eq(containerName), - eq("ip6tables"), - eq("-P"), - eq("FORWARD"), - eq("DROP") - ); - verify(dockerOperations, verificationMode).executeCommandInNetworkNamespace( - eq(containerName), - eq("ip6tables"), - eq("-P"), - eq("OUTPUT"), - eq("ACCEPT") - ); - verify(dockerOperations, verificationMode).executeCommandInNetworkNamespace( - eq(containerName), - eq("ip6tables"), - eq("-A"), - eq("INPUT"), - eq("-m"), - eq("state"), - eq("--state"), - eq("RELATED,ESTABLISHED"), - eq("-j"), - eq("ACCEPT") - ); - verify(dockerOperations, verificationMode).executeCommandInNetworkNamespace( - eq(containerName), - eq("ip6tables"), - eq("-A"), - eq("INPUT"), - eq("-i"), - eq("lo"), - eq("-j"), - eq("ACCEPT") - ); - verify(dockerOperations, verificationMode).executeCommandInNetworkNamespace( - eq(containerName), - eq("ip6tables"), - eq("-A"), - eq("INPUT"), - eq("-p"), - eq("ipv6-icmp"), - eq("-j"), - eq("ACCEPT") - ); - containerAclSpecs.forEach(aclSpec -> verify(dockerOperations, verificationMode).executeCommandInNetworkNamespace( - eq(containerName), - eq("ip6tables"), - eq("-A"), - eq("INPUT"), - eq("-s"), - eq(aclSpec.ipAddress() + "/128"), - eq("-j"), - eq("ACCEPT") - )); - verify(dockerOperations, verificationMode).executeCommandInNetworkNamespace( - eq(containerName), - eq("ip6tables"), - eq("-A"), - eq("INPUT"), - eq("-j"), - eq("REJECT") - ); + eq(containerName), eq("/bin/sh"), eq("-c"), eq(expectedCommand.toString())); } private Container makeContainer(String hostname) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodes.java index 11e9d4521dd..ae4cfbd27f3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/retire/RetireIPv4OnlyNodes.java @@ -35,6 +35,8 @@ public class RetireIPv4OnlyNodes implements RetirementPolicy { return zone.environment() == Environment.prod; } else if (zone.region().equals(RegionName.from("us-central-1"))) { return zone.environment() == Environment.prod; + } else if (zone.region().equals(RegionName.from("ap-southeast-1"))) { + return zone.environment() == Environment.prod; } } diff --git a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp index 223f224aba5..dc352f70706 100644 --- a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp @@ -47,8 +47,7 @@ FieldWriterWrapper::open(const std::string &path, const common::FileHeaderContext &fileHeaderContext) { vespalib::mkdir(path, false); - _writer.earlyOpen(path, 64, 10000, false, schema, indexId, tuneFileWrite); - return _writer.lateOpen(tuneFileWrite, fileHeaderContext); + return _writer.open(path, 64, 10000, false, schema, indexId, tuneFileWrite, fileHeaderContext); } FieldWriterWrapper & diff --git a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp index 72a50d87821..71467519dbd 100644 --- a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp +++ b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp @@ -9,10 +9,8 @@ #include <vespa/searchlib/index/postinglisthandle.h> #include <vespa/searchlib/diskindex/zcposocc.h> #include <vespa/searchlib/diskindex/zcposoccrandread.h> -#include <vespa/searchlib/diskindex/checkpointfile.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/index/schemautil.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/searchlib/diskindex/fieldwriter.h> #include <vespa/searchlib/diskindex/fieldreader.h> #include <vespa/vespalib/io/fileutil.h> @@ -29,7 +27,6 @@ using search::TuneFileRandRead; using search::TuneFileSeqRead; using search::TuneFileSeqWrite; using search::common::FileHeaderContext; -using search::diskindex::CheckPointFile; using search::diskindex::DocIdMapping; using search::diskindex::FieldReader; using search::diskindex::FieldWriter; @@ -48,7 +45,6 @@ using search::index::SchemaUtil; using search::index::schema::CollectionType; using search::index::schema::DataType; using search::queryeval::SearchIterator; -using vespalib::nbostream; using namespace search::index; @@ -91,9 +87,6 @@ makeWordString(uint64_t wordNum) } -typedef std::shared_ptr<FieldReader> FieldReaderSP; -typedef std::shared_ptr<FieldWriter> FieldWriterSP; - class FieldWriterTest : public FastOS_Application { private: @@ -144,10 +137,10 @@ FieldWriterTest::~FieldWriterTest() } -class WrappedFieldWriter : public search::fakedata::CheckPointCallback +class WrappedFieldWriter { public: - FieldWriterSP _fieldWriter; + std::unique_ptr<FieldWriter> _fieldWriter; private: bool _dynamicK; uint32_t _numWordIds; @@ -164,13 +157,8 @@ public: uint32_t docIdLimit); ~WrappedFieldWriter(); - void checkPoint() override; - void earlyOpen(); - void lateOpen(); void open(); void close(); - void writeCheckPoint(); - void readCheckPoint(bool first); }; WrappedFieldWriter::~WrappedFieldWriter() {} @@ -194,32 +182,16 @@ WrappedFieldWriter::WrappedFieldWriter(const vespalib::string &namepref, void -WrappedFieldWriter::earlyOpen() -{ - TuneFileSeqWrite tuneFileWrite; - _fieldWriter.reset(new FieldWriter(_docIdLimit, _numWordIds)); - _fieldWriter->earlyOpen(_namepref, - minSkipDocs, minChunkDocs, _dynamicK, _schema, - _indexId, - tuneFileWrite); -} - - -void -WrappedFieldWriter::lateOpen() +WrappedFieldWriter::open() { TuneFileSeqWrite tuneFileWrite; DummyFileHeaderContext fileHeaderContext; fileHeaderContext.disableFileName(); - _fieldWriter->lateOpen(tuneFileWrite, fileHeaderContext); -} - - -void -WrappedFieldWriter::open() -{ - earlyOpen(); - lateOpen(); + _fieldWriter = std::make_unique<FieldWriter>(_docIdLimit, _numWordIds); + _fieldWriter->open(_namepref, + minSkipDocs, minChunkDocs, _dynamicK, _schema, + _indexId, + tuneFileWrite, fileHeaderContext); } @@ -231,46 +203,10 @@ WrappedFieldWriter::close() } -void -WrappedFieldWriter::writeCheckPoint() -{ - CheckPointFile chkptfile("chkpt"); - nbostream out; - _fieldWriter->checkPointWrite(out); - chkptfile.write(out, DummyFileHeaderContext()); -} - - -void -WrappedFieldWriter::readCheckPoint(bool first) -{ - CheckPointFile chkptfile("chkpt"); - nbostream in; - bool openRes = chkptfile.read(in); - assert(first || openRes); - (void) first; - if (!openRes) - return; - _fieldWriter->checkPointRead(in); - assert(in.empty()); -} - - -void -WrappedFieldWriter::checkPoint() -{ - writeCheckPoint(); - _fieldWriter.reset(); - earlyOpen(); - readCheckPoint(false); - lateOpen(); -} - - -class WrappedFieldReader : public search::fakedata::CheckPointCallback +class WrappedFieldReader { public: - FieldReaderSP _fieldReader; + std::unique_ptr<FieldReader> _fieldReader; private: std::string _namepref; uint32_t _numWordIds; @@ -286,21 +222,15 @@ public: uint32_t docIdLimit); ~WrappedFieldReader(); - void earlyOpen(); - void lateOpen(); void open(); void close(); - void writeCheckPoint(); - void readCheckPoint(bool first); - virtual void checkPoint() override; }; WrappedFieldReader::WrappedFieldReader(const vespalib::string &namepref, uint32_t numWordIds, uint32_t docIdLimit) - : search::fakedata::CheckPointCallback(), - _fieldReader(), + : _fieldReader(), _namepref(dirprefix + namepref), _numWordIds(numWordIds), _docIdLimit(docIdLimit), @@ -323,35 +253,17 @@ WrappedFieldReader::~WrappedFieldReader() { } - -void -WrappedFieldReader::earlyOpen() -{ - TuneFileSeqRead tuneFileRead; - _fieldReader.reset(new FieldReader()); - _fieldReader->earlyOpen(_namepref, tuneFileRead); -} - - void -WrappedFieldReader::lateOpen() +WrappedFieldReader::open() { TuneFileSeqRead tuneFileRead; _wmap.setup(_numWordIds); _dmap.setup(_docIdLimit); + _fieldReader = std::make_unique<FieldReader>(); _fieldReader->setup(_wmap, _dmap); - _fieldReader->lateOpen(_namepref, tuneFileRead); -} - - -void -WrappedFieldReader::open() -{ - earlyOpen(); - lateOpen(); + _fieldReader->open(_namepref, tuneFileRead); } - void WrappedFieldReader::close() { @@ -361,42 +273,6 @@ WrappedFieldReader::close() void -WrappedFieldReader::writeCheckPoint() -{ - CheckPointFile chkptfile("chkpt"); - nbostream out; - _fieldReader->checkPointWrite(out); - chkptfile.write(out, DummyFileHeaderContext()); -} - - -void -WrappedFieldReader::readCheckPoint(bool first) -{ - CheckPointFile chkptfile("chkpt"); - nbostream in; - bool openRes = chkptfile.read(in); - assert(first || openRes); - (void) first; - if (!openRes) - return; - _fieldReader->checkPointRead(in); - assert(in.empty()); -} - - -void -WrappedFieldReader::checkPoint() -{ - writeCheckPoint(); - _fieldReader.reset(); - earlyOpen(); - readCheckPoint(false); - lateOpen(); -} - - -void writeField(FakeWordSet &wordSet, uint32_t docIdLimit, const std::string &namepref, @@ -422,16 +298,11 @@ writeField(FakeWordSet &wordSet, ostate.open(); unsigned int wordNum = 1; - uint32_t checkPointCheck = 0; - uint32_t checkPointInterval = 12227; for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) { for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) { FakeWord &fw = *wordSet._words[wc][wi]; ostate._fieldWriter->newWord(makeWordString(wordNum)); - fw.dump(ostate._fieldWriter, false, - checkPointCheck, - checkPointInterval, - NULL); + fw.dump(*ostate._fieldWriter, false); ++wordNum; } } @@ -450,74 +321,6 @@ writeField(FakeWordSet &wordSet, void -writeFieldCheckPointed(FakeWordSet &wordSet, - uint32_t docIdLimit, - const std::string &namepref, - bool dynamicK) -{ - const char *dynamicKStr = dynamicK ? "true" : "false"; - - FastOS_Time tv; - double before; - double after; - bool first = true; - - LOG(info, - "enter writeFieldCheckPointed, " - "namepref=%s, dynamicK=%s", - namepref.c_str(), - dynamicKStr); - tv.SetNow(); - before = tv.Secs(); - - unsigned int wordNum = 1; - uint32_t checkPointCheck = 0; - uint32_t checkPointInterval = 12227; - for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) { - for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) { - FakeWord &fw = *wordSet._words[wc][wi]; - - WrappedFieldWriter ostate(namepref, - dynamicK, - wordSet.getNumWords(), docIdLimit); - ostate.earlyOpen(); - ostate.readCheckPoint(first); - first = false; - ostate.lateOpen(); - ostate._fieldWriter->newWord(makeWordString(wordNum)); - fw.dump(ostate._fieldWriter, false, - checkPointCheck, - checkPointInterval, - &ostate); - ostate.writeCheckPoint(); - ++wordNum; - } - } - do { - WrappedFieldWriter ostate(namepref, - dynamicK, - wordSet.getNumWords(), docIdLimit); - ostate.earlyOpen(); - ostate.readCheckPoint(first); - ostate.lateOpen(); - ostate.close(); - } while (0); - CheckPointFile dropper("chkpt"); - dropper.remove(); - - tv.SetNow(); - after = tv.Secs(); - LOG(info, - "leave writeFieldCheckPointed, " - "namepref=%s, dynamicK=%s" - " elapsed=%10.6f", - namepref.c_str(), - dynamicKStr, - after - before); -} - - -void readField(FakeWordSet &wordSet, uint32_t docIdLimit, const std::string &namepref, @@ -545,8 +348,6 @@ readField(FakeWordSet &wordSet, TermFieldMatchData mdfield1; unsigned int wordNum = 1; - uint32_t checkPointCheck = 0; - uint32_t checkPointInterval = 12227; for (unsigned int wc = 0; wc < wordSet._words.size(); ++wc) { for (unsigned int wi = 0; wi < wordSet._words[wc].size(); ++wi) { FakeWord &fw = *wordSet._words[wc][wi]; @@ -554,9 +355,8 @@ readField(FakeWordSet &wordSet, TermFieldMatchDataArray tfmda; tfmda.add(&mdfield1); - fw.validate(istate._fieldReader, wordNum, - tfmda, verbose, - checkPointCheck, checkPointInterval, &istate); + fw.validate(*istate._fieldReader, wordNum, + tfmda, verbose); ++wordNum; } } @@ -564,8 +364,6 @@ readField(FakeWordSet &wordSet, istate.close(); tv.SetNow(); after = tv.Secs(); - CheckPointFile dropper("chkpt"); - dropper.remove(); LOG(info, "leave readField, " "namepref=%s, dynamicK=%s" @@ -762,35 +560,23 @@ void testFieldWriterVariants(FakeWordSet &wordSet, uint32_t docIdLimit, bool verbose) { - CheckPointFile dropper("chkpt"); - dropper.remove(); disableSkip(); writeField(wordSet, docIdLimit, "new4", true); readField(wordSet, docIdLimit, "new4", true, verbose); readField(wordSet, docIdLimit, "new4", true, verbose); - writeFieldCheckPointed(wordSet, docIdLimit, "new6", true); writeField(wordSet, docIdLimit, "new5", false); readField(wordSet, docIdLimit, "new5", false, verbose); - writeFieldCheckPointed(wordSet, docIdLimit, "new7", false); enableSkip(); writeField(wordSet, docIdLimit, "newskip4", true); readField(wordSet, docIdLimit, "newskip4", true, verbose); - writeFieldCheckPointed(wordSet, docIdLimit, "newskip6", - true); writeField(wordSet, docIdLimit, "newskip5", false); readField(wordSet, docIdLimit, "newskip5", false, verbose); - writeFieldCheckPointed(wordSet, docIdLimit, "newskip7", - false); enableSkipChunks(); writeField(wordSet, docIdLimit, "newchunk4", true); readField(wordSet, docIdLimit, "newchunk4", true, verbose); - writeFieldCheckPointed(wordSet, docIdLimit, "newchunk6", - true); writeField(wordSet, docIdLimit, "newchunk5", false); readField(wordSet, docIdLimit, "newchunk5",false, verbose); - writeFieldCheckPointed(wordSet, docIdLimit, "newchunk7", - false); disableSkip(); fusionField(wordSet.getNumWords(), docIdLimit, @@ -855,8 +641,6 @@ void testFieldWriterVariantsWithHighLids(FakeWordSet &wordSet, uint32_t docIdLimit, bool verbose) { - CheckPointFile dropper("chkpt"); - dropper.remove(); disableSkip(); writeField(wordSet, docIdLimit, "hlid4", true); readField(wordSet, docIdLimit, "hlid4", true, verbose); diff --git a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp b/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp index 8ef4ad30ea2..36cd15c8ada 100644 --- a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp @@ -5,7 +5,6 @@ #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/memoryindex/documentinverter.h> #include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp b/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp index abab62dfda3..1d066747ef8 100644 --- a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp @@ -2,7 +2,6 @@ #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/repo/fixedtyperepo.h> diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp b/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp index 352b30a6088..16957abe915 100644 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp @@ -5,7 +5,6 @@ #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/memoryindex/fieldinverter.h> #include <vespa/searchlib/memoryindex/urlfieldinverter.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/repo/fixedtyperepo.h> diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index 08f1b8a1e9e..b56f4760ec7 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "compression.h" -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/data/fileheader.h> @@ -9,8 +8,6 @@ namespace search::bitcompression { -using vespalib::nbostream; - uint8_t CodingTables::_log2Table[65536]; CodingTables tables; // Static initializer @@ -129,33 +126,6 @@ EncodeContext64EBase<false>::writeBits(uint64_t data, uint32_t length) } } -void -EncodeContext64Base::checkPointWrite(nbostream &out) -{ - out << _cacheInt << _cacheFree; -} - - -void -EncodeContext64Base::checkPointRead(nbostream &in) -{ - in >> _cacheInt >> _cacheFree; -} - - -void -DecodeContext64Base::checkPointWrite(nbostream &out) -{ - (void) out; -} - - -void -DecodeContext64Base::checkPointRead(nbostream &in) -{ - (void) in; -} - namespace { vespalib::string noFeatures = "NoFeatures"; diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h index a0cba703e65..954afc60a1a 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h @@ -9,7 +9,6 @@ namespace vespalib { -class nbostream; class GenericHeader; } @@ -830,9 +829,6 @@ public: other._cacheFree = _cacheFree; } - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; - uint64_t getWriteOffset() const { return _fileWriteBias + (reinterpret_cast<unsigned long>(_valI) << 3) - _cacheFree; } @@ -1269,9 +1265,6 @@ public: return (_preRead == 0) ? 0 : 64 - _preRead; } - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; - static int64_t convertToSigned(uint64_t val) { if ((val & 1) != 0) return - (val >> 1) - 1; diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp index fca6c749cfb..49d5ae92ec8 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp @@ -3,14 +3,11 @@ #include "compression.h" #include "countcompression.h" #include <vespa/searchlib/index/postinglistcounts.h> -#include <vespa/vespalib/objects/nbostream.h> namespace search { namespace bitcompression { -using vespalib::nbostream; - #define K_VALUE_COUNTFILE_LASTDOCID 22 #define K_VALUE_COUNTFILE_NUMCHUNKS 1 #define K_VALUE_COUNTFILE_CHUNKNUMDOCS 18 @@ -19,24 +16,6 @@ using vespalib::nbostream; void -PostingListCountFileDecodeContext::checkPointWrite(nbostream &out) -{ - ParentClass::checkPointWrite(out); - out << _avgBitsPerDoc << _minChunkDocs << _docIdLimit << _numWordIds; - out << _minWordNum; -} - - -void -PostingListCountFileDecodeContext::checkPointRead(nbostream &in) -{ - ParentClass::checkPointRead(in); - in >> _avgBitsPerDoc >> _minChunkDocs >> _docIdLimit >> _numWordIds; - in >> _minWordNum; -} - - -void PostingListCountFileDecodeContext:: readCounts(PostingListCounts &counts) { @@ -145,24 +124,6 @@ copyParams(const PostingListCountFileDecodeContext &rhs) void -PostingListCountFileEncodeContext::checkPointWrite(nbostream &out) -{ - ParentClass::checkPointWrite(out); - out << _avgBitsPerDoc << _minChunkDocs << _docIdLimit << _numWordIds; - out << _minWordNum; -} - - -void -PostingListCountFileEncodeContext::checkPointRead(nbostream &in) -{ - ParentClass::checkPointRead(in); - in >> _avgBitsPerDoc >> _minChunkDocs >> _docIdLimit >> _numWordIds; - in >> _minWordNum; -} - - -void PostingListCountFileEncodeContext:: writeCounts(const PostingListCounts &counts) { diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.h b/searchlib/src/vespa/searchlib/bitcompression/countcompression.h index 4a514e58e2a..fc448ba1c30 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.h @@ -33,8 +33,6 @@ public: { } - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; void readCounts(PostingListCounts &counts); void readWordNum(uint64_t &wordNum); @@ -67,8 +65,6 @@ public: { } - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; void writeCounts(const PostingListCounts &counts); void writeWordNum(uint64_t wordNum); diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp index b3cba84d575..82110d354d3 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp @@ -5,7 +5,6 @@ #include "countcompression.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/dictionaryfile.h> -#include <vespa/vespalib/objects/nbostream.h> #include <sstream> #include <vespa/log/log.h> @@ -71,60 +70,6 @@ operator<<(std::ostream &stream, const index::PostingListCounts &counts) return stream; } -vespalib::nbostream & -operator<<(vespalib::nbostream &stream, - const PageDict4StartOffset &startOffset) -{ - stream << startOffset._fileOffset << startOffset._accNumDocs; - return stream; -} - -vespalib::nbostream & -operator>>(vespalib::nbostream &stream, PageDict4StartOffset &startOffset) -{ - stream >> startOffset._fileOffset >> startOffset._accNumDocs; - return stream; -} - - -vespalib::nbostream & -operator<<(vespalib::nbostream &stream, - const PageDict4SSReader::L7Entry &l7Entry) -{ - stream << l7Entry._l7Word << l7Entry._l7StartOffset << l7Entry._l7WordNum; - stream << l7Entry._l6Offset << l7Entry._sparsePageNum << l7Entry._pageNum; - stream << l7Entry._l7Ref; - return stream; -} - - -vespalib::nbostream & -operator>>(vespalib::nbostream &stream, - PageDict4SSReader::L7Entry &l7Entry) -{ - stream >> l7Entry._l7Word >> l7Entry._l7StartOffset >> l7Entry._l7WordNum; - stream >> l7Entry._l6Offset >> l7Entry._sparsePageNum >> l7Entry._pageNum; - stream >> l7Entry._l7Ref; - return stream; -} - - -vespalib::nbostream & -operator<<(vespalib::nbostream &stream, - const PageDict4SSReader::OverflowRef &oref) -{ - stream << oref._wordNum << oref._l7Ref; - return stream; -} - - -vespalib::nbostream & -operator>>(vespalib::nbostream &stream, PageDict4SSReader::OverflowRef &oref) -{ - stream >> oref._wordNum >> oref._l7Ref; - return stream; -} - typedef index::PostingListCounts Counts; typedef PageDict4StartOffset StartOffset; @@ -360,28 +305,6 @@ PageDict4SSWriter::flush() } -void -PageDict4SSWriter::checkPointWrite(vespalib::nbostream &out) -{ - out << _l6Word; - out << _l6StartOffset; - out << _l6PageNum; - out << _l6SparsePageNum; - out << _l6WordNum; -} - - -void -PageDict4SSWriter::checkPointRead(vespalib::nbostream &in) -{ - in >> _l6Word; - in >> _l6StartOffset; - in >> _l6PageNum; - in >> _l6SparsePageNum; - in >> _l6WordNum; -} - - PageDict4SPWriter::PageDict4SPWriter(SSWriter &ssWriter, EC &spe) : _eL3(), @@ -725,48 +648,6 @@ PageDict4SPWriter::addL5Skip(size_t &lcp) } -void -PageDict4SPWriter::checkPointWrite(vespalib::nbostream &out) -{ - _wcL3.checkPointWrite(out); - _wcL4.checkPointWrite(out); - _wcL5.checkPointWrite(out); - out << _l3Word << _l4Word << _l5Word << _l6Word; - out << _l3WordOffset << _l4WordOffset << _l5WordOffset; - out << _l3StartOffset << _l4StartOffset << _l5StartOffset << _l6StartOffset; - out << _l3WordNum << _l4WordNum << _l5WordNum << _l6WordNum; - out << _curL3OffsetL4 << _curL3OffsetL5 << _curL4OffsetL5; - out << _headerSize; - out << _l3Entries; - out << _l4StrideCheck << _l5StrideCheck; - out << _l3Size << _l4Size << _l5Size; - out << _prevL3Size << _prevL4Size << _prevL5Size << _prevWordsSize; - out << _sparsePageNum << _l3PageNum; - out << _words; -} - - -void -PageDict4SPWriter::checkPointRead(vespalib::nbostream &in) -{ - _wcL3.checkPointRead(in); - _wcL4.checkPointRead(in); - _wcL5.checkPointRead(in); - in >> _l3Word >> _l4Word >> _l5Word >> _l6Word; - in >> _l3WordOffset >> _l4WordOffset >> _l5WordOffset; - in >> _l3StartOffset >> _l4StartOffset >> _l5StartOffset >> _l6StartOffset; - in >> _l3WordNum >> _l4WordNum >> _l5WordNum >> _l6WordNum; - in >> _curL3OffsetL4 >> _curL3OffsetL5 >> _curL4OffsetL5; - in >> _headerSize; - in >> _l3Entries; - in >> _l4StrideCheck >> _l5StrideCheck; - in >> _l3Size >> _l4Size >> _l5Size; - in >> _prevL3Size >> _prevL4Size >> _prevL5Size >> _prevWordsSize; - in >> _sparsePageNum >> _l3PageNum; - in >> _words; -} - - PageDict4PWriter::PageDict4PWriter(SPWriter &spWriter, EC &pe) : _eCounts(), @@ -1148,52 +1029,6 @@ PageDict4PWriter::addL2Skip(size_t &lcp) } -void -PageDict4PWriter::checkPointWrite(vespalib::nbostream &out) -{ - _wcCounts.checkPointWrite(out); - _wcL1.checkPointWrite(out); - _wcL2.checkPointWrite(out); - out << _countsWord << _l1Word << _l2Word << _l3Word; - out << _pendingCountsWord; - out << _countsWordOffset << _l1WordOffset << _l2WordOffset; - out << _countsStartOffset << _l1StartOffset << _l2StartOffset; - out << _l3StartOffset; - out << _curCountOffsetL1 << _curCountOffsetL2 << _curL1OffsetL2; - out << _headerSize; - out << _countsEntries; - out << _l1StrideCheck << _l2StrideCheck; - out << _countsSize << _l1Size << _l2Size; - out << _prevL1Size << _prevL2Size; - out << _pageNum; - out << _l3WordNum << _wordNum; - out << _words; -} - - -void -PageDict4PWriter::checkPointRead(vespalib::nbostream &in) -{ - _wcCounts.checkPointRead(in); - _wcL1.checkPointRead(in); - _wcL2.checkPointRead(in); - in >> _countsWord >> _l1Word >> _l2Word >> _l3Word; - in >> _pendingCountsWord; - in >> _countsWordOffset >> _l1WordOffset >> _l2WordOffset; - in >> _countsStartOffset >> _l1StartOffset >> _l2StartOffset; - in >> _l3StartOffset; - in >> _curCountOffsetL1 >> _curCountOffsetL2 >> _curL1OffsetL2; - in >> _headerSize; - in >> _countsEntries; - in >> _l1StrideCheck >> _l2StrideCheck; - in >> _countsSize >> _l1Size >> _l2Size; - in >> _prevL1Size >> _prevL2Size; - in >> _pageNum; - in >> _l3WordNum >> _wordNum; - in >> _words; -} - - PageDict4SSLookupRes:: PageDict4SSLookupRes() : _l6Word(), @@ -1660,34 +1495,6 @@ lookupOverflow(uint64_t wordNum) const } -void -PageDict4SSReader::checkPointWrite(vespalib::nbostream &out) -{ - out << _ssFileBitLen << _ssStartOffset; - out << _l7; - _ssd.checkPointWrite(out); - out << _spFileBitLen << _pFileBitLen; - out << _spStartOffset << _pStartOffset; - out << _spFirstPageNum << _spFirstPageOffset; - out << _pFirstPageNum << _pFirstPageOffset; - out << _overflows; -} - - -void -PageDict4SSReader::checkPointRead(vespalib::nbostream &in) -{ - in >> _ssFileBitLen >> _ssStartOffset; - in >> _l7; - _ssd.checkPointRead(in); - in >> _spFileBitLen >> _pFileBitLen; - in >> _spStartOffset >> _pStartOffset; - in >> _spFirstPageNum >> _spFirstPageOffset; - in >> _pFirstPageNum >> _pFirstPageOffset; - in >> _overflows; -} - - PageDict4SPLookupRes:: PageDict4SPLookupRes() : _l3Word(), @@ -2512,68 +2319,6 @@ PageDict4Reader::readOverflowCounts(vespalib::string &word, _startOffset.adjust(counts); } -void -PageDict4Reader::checkPointWrite(vespalib::nbostream &out) -{ - out << _countsResidue; - out << _overflowPage; - out << _counts; - size_t ccOff = _cc - _counts.begin(); - size_t ceOff = _ce - _counts.begin(); - assert(ceOff == _counts.size()); - out << ccOff << ceOff; - out << _words; - size_t wcOff = _wc - _words.begin(); - size_t weOff = _we - _words.begin(); - assert(weOff = _words.size()); - out << wcOff << weOff; - out << _lastWord; - out << _lastSSWord; - out << _l3Residue; - out << _spwords; - size_t spwcOff = _spwc - _spwords.begin(); - size_t spweOff = _spwe - _spwords.begin(); - assert(spweOff == _spwords.size()); - out << spwcOff << spweOff; - _ssd.checkPointWrite(out); - out << _ssd.getReadOffset(); - out << _wordNum; -} - -void -PageDict4Reader::checkPointRead(vespalib::nbostream &in) -{ - in >> _countsResidue; - in >> _overflowPage; - in >> _counts; - size_t ccOff; - size_t ceOff; - in >> ccOff >> ceOff; - _cc = _counts.begin() + ccOff; - _ce = _counts.begin() + ceOff; - in >> _words; - size_t wcOff; - size_t weOff; - in >> wcOff >> weOff; - _wc = _words.begin() + wcOff; - _we = _words.begin() + weOff; - in >> _lastWord; - in >> _lastSSWord; - in >> _l3Residue; - in >> _spwords; - size_t spwcOff; - size_t spweOff; - in >> spwcOff >> spweOff; - _spwc = _spwords.begin() + spwcOff; - _spwe = _spwords.begin() + spweOff; - _ssd.checkPointRead(in); - int64_t ssReadOffset; - in >> ssReadOffset; - const ComprBuffer &sscb = _ssReader._cb; - setDecoderPosition(_ssd, sscb, ssReadOffset); - in >> _wordNum; -} - } // namespace bitcompression } // namespace search diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h index 2a4cd143cfa..47f2354bcc6 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h +++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h @@ -160,13 +160,6 @@ public: void flush(); - - - void - checkPointWrite(vespalib::nbostream &out); - - void - checkPointRead(vespalib::nbostream &in); }; @@ -307,12 +300,6 @@ public: { _ssWriter.addOverflowCounts(word, counts, startOffset, wordNum); } - - void - checkPointWrite(vespalib::nbostream &out); - - void - checkPointRead(vespalib::nbostream &in); }; /* @@ -447,12 +434,6 @@ public: { return _wordNum - 1; } - - void - checkPointWrite(vespalib::nbostream &out); - - void - checkPointRead(vespalib::nbostream &in); }; @@ -534,12 +515,6 @@ public: { return _l7Word < word; } - - friend vespalib::nbostream & - operator<<(vespalib::nbostream &stream, const L7Entry &l7Entry); - - friend vespalib::nbostream & - operator>>(vespalib::nbostream &stream, L7Entry &l7Entry); }; class OverflowRef @@ -565,12 +540,6 @@ public: { return _wordNum < wordNum; } - - friend vespalib::nbostream & - operator<<(vespalib::nbostream &stream, const OverflowRef &oref); - - friend vespalib::nbostream & - operator>>(vespalib::nbostream &stream, OverflowRef &oref); }; ComprBuffer _cb; @@ -617,12 +586,6 @@ public: { return _ssd; } - - void - checkPointWrite(vespalib::nbostream &out); - - void - checkPointRead(vespalib::nbostream &in); }; @@ -752,12 +715,6 @@ public: void readOverflowCounts(vespalib::string &word, Counts &counts); - - void - checkPointWrite(vespalib::nbostream &out); - - void - checkPointRead(vespalib::nbostream &in); }; } // namespace bitcompression diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp index 4931455da8d..dcee48aed1a 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp @@ -5,7 +5,6 @@ namespace search { -using vespalib::nbostream; using vespalib::GenerationHeldBase; using vespalib::GenerationHeldAlloc; using vespalib::GenerationHolder; diff --git a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt index 9b6bf497c9a..3619affb54e 100644 --- a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_library(searchlib_diskindex OBJECT bitvectorfile.cpp bitvectoridxfile.cpp bitvectorkeyscope.cpp - checkpointfile.cpp dictionarywordreader.cpp diskindex.cpp disktermblueprint.cpp diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp index defacdb05d7..374c6a8c7e6 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp @@ -4,12 +4,10 @@ #include <vespa/searchlib/index/bitvectorkeys.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/fileheadercontext.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/fileheader.h> namespace search::diskindex { -using vespalib::nbostream; using search::index::BitVectorWordSingleKey; using search::common::FileHeaderContext; @@ -45,24 +43,6 @@ BitVectorFileWrite::~BitVectorFileWrite() void -BitVectorFileWrite::checkPointWrite(nbostream &out) -{ - flush(); - Parent::checkPointWriteCommon(out); - out << _datHeaderLen; - sync(); -} - - -void -BitVectorFileWrite::checkPointRead(nbostream &in) -{ - Parent::checkPointRead(in); - in >> _datHeaderLen; -} - - -void BitVectorFileWrite::open(const vespalib::string &name, uint32_t docIdLimit, const TuneFileSeqWrite &tuneFileWrite, @@ -198,33 +178,4 @@ BitVectorFileWrite::close() BitVectorCandidate::~BitVectorCandidate() { } -void -BitVectorCandidate::checkPointWrite(nbostream &out) -{ - uint32_t docIdLimit = _bv->size(); - out << docIdLimit << _numDocs << _bitVectorLimit; - out.saveVector(_array); - if (getCrossedBitVectorLimit()) - out << *_bv; -} - - -void -BitVectorCandidate::checkPointRead(nbostream &in) -{ - uint32_t docIdLimit = _bv->size(); - uint32_t checkDocIdLimit; - uint32_t checkBitVectorLimit; - in >> checkDocIdLimit >> _numDocs >> checkBitVectorLimit; - assert(checkDocIdLimit == docIdLimit); - (void) docIdLimit; - assert(checkBitVectorLimit == _bitVectorLimit); - in.restoreVector(_array); - if (getCrossedBitVectorLimit()) { - in >> *_bv; - } else { - _bv->clear(); - } -} - } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h index 1b7b522bc52..893b792d96b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h @@ -37,21 +37,6 @@ public: ~BitVectorFileWrite(); - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - void - checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void - checkPointRead(vespalib::nbostream &in); - void open(const vespalib::string &name, uint32_t docIdLimit, const TuneFileSeqWrite &tuneFileWrite, @@ -181,21 +166,6 @@ public: { return *_bv; } - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - void - checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void - checkPointRead(vespalib::nbostream &in); }; diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp index 856c527a8b5..a4eca7feac9 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp @@ -4,12 +4,10 @@ #include <vespa/searchlib/index/bitvectorkeys.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/fileheadercontext.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/fileheader.h> namespace search::diskindex { -using vespalib::nbostream; using search::index::BitVectorWordSingleKey; using search::common::FileHeaderContext; @@ -55,35 +53,6 @@ BitVectorIdxFileWrite::idxSize() const void -BitVectorIdxFileWrite::checkPointWriteCommon(nbostream &out) -{ - out << _scope; - out << _docIdLimit << _numKeys; - out << _idxHeaderLen; -} - - -void -BitVectorIdxFileWrite::checkPointWrite(nbostream &out) -{ - flush(); - checkPointWriteCommon(out); - sync(); -} - - -void -BitVectorIdxFileWrite::checkPointRead(nbostream &in) -{ - BitVectorKeyScope checkScope; - in >> checkScope; - assert(checkScope == _scope); - in >> _docIdLimit >> _numKeys; - in >> _idxHeaderLen; -} - - -void BitVectorIdxFileWrite::open(const vespalib::string &name, uint32_t docIdLimit, const TuneFileSeqWrite &tuneFileWrite, diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h index 0ab40d9c4a3..8d704d1bffe 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h @@ -8,13 +8,6 @@ #include <vespa/vespalib/stllike/string.h> #include "bitvectorkeyscope.h" -namespace vespalib -{ - -class nbostream; - -} - namespace search { @@ -49,7 +42,6 @@ protected: BitVectorKeyScope _scope; uint64_t idxSize() const; - void checkPointWriteCommon(vespalib::nbostream &out); void syncCommon(); public: @@ -57,21 +49,6 @@ public: ~BitVectorIdxFileWrite(); - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - void - checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void - checkPointRead(vespalib::nbostream &in); - void open(const vespalib::string &name, uint32_t docIdLimit, const TuneFileSeqWrite &tuneFileWrite, diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.cpp index 2e02b092ddd..ee18383debd 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bitvectorkeyscope.h" -#include <vespa/vespalib/objects/nbostream.h> #include <cassert> using search::diskindex::BitVectorKeyScope; @@ -19,46 +18,3 @@ const char *getBitVectorKeyScopeSuffix(BitVectorKeyScope scope) } } - - -namespace { - -uint8_t -getVal(BitVectorKeyScope scope) -{ - switch (scope) { - case BitVectorKeyScope::SHARED_WORDS: - return 0u; - default: - return 1u; - } -} - - -const BitVectorKeyScope scopes[] = { BitVectorKeyScope::SHARED_WORDS, - BitVectorKeyScope::PERFIELD_WORDS }; - -} - - -namespace vespalib { - -nbostream & -operator<<(nbostream &stream, const BitVectorKeyScope &scope) -{ - uint8_t val = getVal(scope); - stream << val; - return stream; -} - -nbostream & -operator>>(nbostream &stream, BitVectorKeyScope &scope) -{ - uint8_t val; - stream >> val; - assert(val < sizeof(scopes) / sizeof(scopes[0])); - scope = scopes[val]; - return stream; -} - -} diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.h index 1c931225cc0..7b2af6adb9d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.h @@ -3,14 +3,6 @@ #pragma once -namespace vespalib -{ - -class nbostream; - -} - - namespace search { @@ -28,16 +20,3 @@ const char *getBitVectorKeyScopeSuffix(BitVectorKeyScope scope); } } - -namespace vespalib -{ - -nbostream & -operator<<(nbostream &stream, - const search::diskindex::BitVectorKeyScope &scope); - -nbostream & -operator>>(nbostream &stream, - search::diskindex::BitVectorKeyScope &scope); - -} diff --git a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp b/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp deleted file mode 100644 index 0324f00f63c..00000000000 --- a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "checkpointfile.h" -#include <vespa/vespalib/data/fileheader.h> -#include <vespa/searchlib/common/fileheadercontext.h> -#include <cassert> - -#include <vespa/log/log.h> -LOG_SETUP(".diskindex.checkpointfile"); - -using vespalib::getLastErrorString; - -namespace search::diskindex { - -using common::FileHeaderContext; - -CheckPointFile::CheckPointFile(const vespalib::string &name) - : _file(), - _name(name), - _nameNew(name + ".NEW"), - _nameNewNew(name + ".NEW.NEW"), - _writeOpened(false), - _headerLen(0u) -{ } - - -CheckPointFile::~CheckPointFile() -{ - close(); -} - - -void -CheckPointFile::writeOpen(const FileHeaderContext &fileHeaderContext) -{ - FastOS_File::Delete(_nameNewNew.c_str()); - _file.OpenWriteOnly(_nameNewNew.c_str()); - _writeOpened = true; - makeHeader(fileHeaderContext); -} - - -bool -CheckPointFile::readOpen() -{ - bool openres; - - openres = _file.OpenReadOnly(_name.c_str()); - if (!openres) { - bool renameres = FastOS_File::Rename(_nameNew.c_str(), - _name.c_str()); - if (!renameres) - return false; - openres = _file.OpenReadOnly(_name.c_str()); - if (!openres) - return false; - } - _headerLen = readHeader(); - return true; -} - - -void -CheckPointFile::close() -{ - if (_writeOpened) { - _file.Sync(); - } - _file.Close(); - if (_writeOpened) { - updateHeader(); - rename1(); - rename2(); - } - _writeOpened = false; -} - - -void -CheckPointFile::rename1() -{ - FastOS_File::Delete(_nameNew.c_str()); - bool renameres = FastOS_File::Rename(_nameNewNew.c_str(), - _nameNew.c_str()); - if (!renameres) { - LOG(error, "FATAL: rename %s -> %s failed: %s", - _nameNewNew.c_str(), _nameNew.c_str(), getLastErrorString().c_str()); - abort(); - } -} - - -void -CheckPointFile::rename2() -{ - FastOS_File::Delete(_name.c_str()); - bool renameres = FastOS_File::Rename(_nameNew.c_str(), _name.c_str()); - if (!renameres) { - LOG(error, "FATAL: rename %s -> %s failed: %s", - _nameNew.c_str(), _name.c_str(), getLastErrorString().c_str()); - abort(); - } -} - - -void -CheckPointFile::remove() -{ - FastOS_File::Delete(_nameNew.c_str()); - FastOS_File::Delete(_name.c_str()); -} - - - -void -CheckPointFile::write(vespalib::nbostream &buf, - const FileHeaderContext &fileHeaderContext) -{ - writeOpen(fileHeaderContext); - _file.WriteBuf(buf.peek(), buf.size()); - close(); -} - - -bool -CheckPointFile::read(vespalib::nbostream &buf) -{ - if (!readOpen()) - return false; - size_t sz = _file.GetSize() - _headerLen; - - std::vector<char> tmp(sz); - _file.ReadBuf(&tmp[0], sz); - buf.clear(); - buf.write(&tmp[0], sz); - std::vector<char>().swap(tmp); - close(); - return true; -} - - -void -CheckPointFile::makeHeader(const FileHeaderContext &fileHeaderContext) -{ - vespalib::FileHeader header; - - typedef vespalib::GenericHeader::Tag Tag; - fileHeaderContext.addTags(header, _file.GetFileName()); - header.putTag(Tag("frozen", 0)); - header.putTag(Tag("desc", "Check point file")); - header.writeFile(_file); -} - - -void -CheckPointFile::updateHeader() -{ - vespalib::FileHeader h; - FastOS_File f; - f.OpenReadWrite(_nameNewNew.c_str()); - h.readFile(f); - FileHeaderContext::setFreezeTime(h); - typedef vespalib::GenericHeader::Tag Tag; - h.putTag(Tag("frozen", 1)); - h.rewriteFile(f); - f.Sync(); - f.Close(); -} - - -uint32_t -CheckPointFile::readHeader() -{ - vespalib::FileHeader h; - uint32_t headerLen = h.readFile(_file); - _file.SetPosition(headerLen); - assert(h.hasTag("frozen")); - assert(h.getTag("frozen").asInteger() != 0); - return headerLen; -} - - -} diff --git a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.h b/searchlib/src/vespa/searchlib/diskindex/checkpointfile.h deleted file mode 100644 index 7ea18132b08..00000000000 --- a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/objects/nbostream.h> -#include <vespa/fastos/file.h> - -namespace search { - -namespace common { class FileHeaderContext; } - -namespace diskindex { - -class CheckPointFile -{ -public: - FastOS_File _file; - vespalib::string _name; - vespalib::string _nameNew; - vespalib::string _nameNewNew; - bool _writeOpened; - uint32_t _headerLen; - - void writeOpen(const common::FileHeaderContext &fileHeaderContext); - bool readOpen(); - void close(); - void rename1(); - void rename2(); - void remove(); - void makeHeader(const common::FileHeaderContext &fileHeaderContext); - void updateHeader(); - uint32_t readHeader(); -public: - CheckPointFile(const CheckPointFile &) = delete; - CheckPointFile & operator = (const CheckPointFile &) = delete; - CheckPointFile(const vespalib::string &name); - ~CheckPointFile(); - - void write(vespalib::nbostream &buf, const common::FileHeaderContext &fileHeaderContext); - bool read(vespalib::nbostream &buf); -}; - - -} // namespace diskindex - -} // namespace search - diff --git a/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp b/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp index 84066aa4e65..70f7bdfea10 100644 --- a/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docidmapper.h" -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/searchlib/common/documentsummary.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/fastlib/io/bufferedfile.h> diff --git a/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp b/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp index 89c801f04b5..c6f9cc757fd 100644 --- a/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp @@ -7,7 +7,6 @@ #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/postinglistcountfile.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".diskindex.extposocc"); diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp index c1964906f85..cffc2e09ef8 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp @@ -5,7 +5,6 @@ #include "extposocc.h" #include "pagedict4file.h" #include <vespa/vespalib/util/error.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".diskindex.fieldreader"); @@ -36,7 +35,6 @@ FieldReader::FieldReader() _oldWordNum(noWordNumHigh()), _residue(0u), _docIdLimit(0u), - _checkPointResume(false), _word() { } @@ -108,8 +106,8 @@ FieldReader::setup(const WordNumMapping &wordNumMapping, bool -FieldReader::earlyOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) +FieldReader::open(const vespalib::string &prefix, + const TuneFileSeqRead &tuneFileRead) { vespalib::string name = prefix + "posocc.dat.compressed"; FastOS_StatInfo statInfo; @@ -124,28 +122,20 @@ FieldReader::earlyOpen(const vespalib::string &prefix, return false; } - _dictFile.reset(new search::diskindex::PageDict4FileSeqRead); + _dictFile = std::make_unique<PageDict4FileSeqRead>(); PostingListParams featureParams; - _oldposoccfile.reset(search::diskindex::makePosOccRead(name, - _dictFile.get(), - dynamicKPosOccFormat, - featureParams, - tuneFileRead)); - return true; -} - - -bool -FieldReader::lateOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) -{ + _oldposoccfile.reset(makePosOccRead(name, + _dictFile.get(), + dynamicKPosOccFormat, + featureParams, + tuneFileRead)); vespalib::string cname = prefix + "dictionary"; - vespalib::string name = prefix + "posocc.dat.compressed"; if (!_dictFile->open(cname, tuneFileRead)) { LOG(error, "Could not open posocc count file %s for read", cname.c_str()); + return false; } // open posocc.dat @@ -153,28 +143,16 @@ FieldReader::lateOpen(const vespalib::string &prefix, LOG(error, "Could not open posocc file %s for read", name.c_str()); + return false; } - if (!_checkPointResume) { - _oldWordNum = noWordNum(); - _wordNum = _oldWordNum; - PostingListParams params; - _oldposoccfile->getParams(params); - params.get("docIdLimit", _docIdLimit); - } + _oldWordNum = noWordNum(); + _wordNum = _oldWordNum; + PostingListParams params; + _oldposoccfile->getParams(params); + params.get("docIdLimit", _docIdLimit); return true; } - -bool -FieldReader::open(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) -{ - if (!earlyOpen(prefix, tuneFileRead)) - return false; - return lateOpen(prefix, tuneFileRead); -} - - bool FieldReader::close() { @@ -204,29 +182,6 @@ FieldReader::close() void -FieldReader::checkPointWrite(vespalib::nbostream &out) -{ - out << _wordNum << _oldWordNum; - out << _residue << _docIdAndFeatures; - out << _docIdLimit; - out << _word; - _oldposoccfile->checkPointWrite(out); - _dictFile->checkPointWrite(out); -} - -void -FieldReader::checkPointRead(vespalib::nbostream &in) -{ - in >> _wordNum >> _oldWordNum; - in >> _residue >> _docIdAndFeatures; - in >> _docIdLimit; - in >> _word; - _oldposoccfile->checkPointRead(in); - _dictFile->checkPointRead(in); - _checkPointResume = true; -} - -void FieldReader::setFeatureParams(const PostingListParams ¶ms) { _oldposoccfile->setFeatureParams(params); @@ -261,26 +216,6 @@ FieldReaderEmpty::FieldReaderEmpty(const IndexIterator &index) bool -FieldReaderEmpty::earlyOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) -{ - (void) prefix; - (void) tuneFileRead; - return true; -} - - -bool -FieldReaderEmpty::lateOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) -{ - (void) prefix; - (void) tuneFileRead; - return true; -} - - -bool FieldReaderEmpty::open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead) { diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h index 3d724f31b41..d55aa39d491 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h @@ -56,7 +56,6 @@ protected: uint64_t _oldWordNum; uint32_t _residue; uint32_t _docIdLimit; - bool _checkPointResume; vespalib::string _word; static uint64_t @@ -117,31 +116,11 @@ public: const DocIdMapping &docIdMapping); virtual bool - earlyOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead); - - virtual bool - lateOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead); - - virtual bool open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead); virtual bool close(); - /* - * To be called between words, not in the middle of one. - */ - virtual void - checkPointWrite(vespalib::nbostream &out); - - /* - * To be called after earlyOpen() but before afterOpen(). - */ - virtual void - checkPointRead(vespalib::nbostream &in); - virtual void setFeatureParams(const PostingListParams ¶ms); @@ -172,14 +151,6 @@ public: FieldReaderEmpty(const IndexIterator &index); virtual bool - earlyOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) override; - - virtual bool - lateOpen(const vespalib::string &prefix, - const TuneFileSeqRead &tuneFileRead) override; - - virtual bool open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead) override; @@ -213,4 +184,3 @@ public: } // namespace diskindex } // namespace search - diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp index 9f03c06cf56..6b66568ef7a 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp @@ -5,14 +5,12 @@ #include "extposocc.h" #include "pagedict4file.h" #include <vespa/vespalib/util/error.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".diskindex.fieldwriter"); namespace search::diskindex { -using vespalib::nbostream; using vespalib::getLastErrorString; using common::FileHeaderContext; @@ -34,14 +32,15 @@ FieldWriter::FieldWriter(uint32_t docIdLimit, FieldWriter::~FieldWriter() { } -void -FieldWriter::earlyOpen(const vespalib::string &prefix, - uint32_t minSkipDocs, - uint32_t minChunkDocs, - bool dynamicKPosOccFormat, - const Schema &schema, - const uint32_t indexId, - const TuneFileSeqWrite &tuneFileWrite) +bool +FieldWriter::open(const vespalib::string &prefix, + uint32_t minSkipDocs, + uint32_t minChunkDocs, + bool dynamicKPosOccFormat, + const Schema &schema, + const uint32_t indexId, + const TuneFileSeqWrite &tuneFileWrite, + const FileHeaderContext &fileHeaderContext) { _prefix = prefix; vespalib::string name = prefix + "posocc.dat.compressed"; @@ -64,7 +63,7 @@ FieldWriter::earlyOpen(const vespalib::string &prefix, params.set("minChunkDocs", minChunkDocs); } - _dictFile.reset(new PageDict4FileSeqWrite); + _dictFile = std::make_unique<PageDict4FileSeqWrite>(); _dictFile->setParams(countParams); _posoccfile.reset(diskindex::makePosOccWrite(name, @@ -75,15 +74,7 @@ FieldWriter::earlyOpen(const vespalib::string &prefix, schema, indexId, tuneFileWrite)); -} - - -bool -FieldWriter::lateOpen(const TuneFileSeqWrite &tuneFileWrite, - const FileHeaderContext &fileHeaderContext) -{ vespalib::string cname = _prefix + "dictionary"; - vespalib::string name = _prefix + "posocc.dat.compressed"; // Open output dictionary file if (!_dictFile->open(cname, tuneFileWrite, fileHeaderContext)) { @@ -181,36 +172,6 @@ FieldWriter::close() void -FieldWriter::checkPointWrite(nbostream &out) -{ - out << _wordNum << _prevDocId; - out << _docIdLimit << _numWordIds; - out << _compactWordNum << _word; - _posoccfile->checkPointWrite(out); - _dictFile->checkPointWrite(out); - _bvc.checkPointWrite(out); - _bmapfile.checkPointWrite(out); -} - - -void -FieldWriter::checkPointRead(nbostream &in) -{ - in >> _wordNum >> _prevDocId; - uint32_t checkDocIdLimit = 0; - uint64_t checkNumWordIds = 0; - in >> checkDocIdLimit >> checkNumWordIds; - assert(checkDocIdLimit == _docIdLimit); - assert(checkNumWordIds == _numWordIds); - in >> _compactWordNum >> _word; - _posoccfile->checkPointRead(in); - _dictFile->checkPointRead(in); - _bvc.checkPointRead(in); - _bmapfile.checkPointRead(in); -} - - -void FieldWriter::setFeatureParams(const PostingListParams ¶ms) { _posoccfile->setFeatureParams(params); diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h index 1cd53b99bce..8f879360ea5 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h @@ -8,8 +8,6 @@ #include <vespa/searchlib/bitcompression/countcompression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> -namespace vespalib { class nbostream; } - namespace search { namespace diskindex { @@ -73,24 +71,13 @@ public: uint64_t getSparseWordNum() const { return _wordNum; } - void earlyOpen(const vespalib::string &prefix, uint32_t minSkipDocs, uint32_t minChunkDocs, - bool dynamicKPosOccFormat, const Schema &schema, uint32_t indexId, - const TuneFileSeqWrite &tuneFileWrite); - - bool lateOpen(const TuneFileSeqWrite &tuneFileWrite, - const search::common::FileHeaderContext &fileHeaderContext); + bool open(const vespalib::string &prefix, uint32_t minSkipDocs, uint32_t minChunkDocs, + bool dynamicKPosOccFormat, const Schema &schema, uint32_t indexId, + const TuneFileSeqWrite &tuneFileWrite, + const search::common::FileHeaderContext &fileHeaderContext); bool close(); - /* - * To be called between words, not in the middle of one. - */ - void checkPointWrite(vespalib::nbostream &out); - - /* - * To be called after earlyOpen() but before afterOpen(). - */ - void checkPointRead(vespalib::nbostream &in); void setFeatureParams(const PostingListParams ¶ms); void getFeatureParams(PostingListParams ¶ms); static void remove(const vespalib::string &prefix); diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index 97d21caef96..5dc5af68a38 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -314,16 +314,14 @@ Fusion::openFieldWriter(const SchemaUtil::IndexIterator &index, { vespalib::string dir = _outDir + "/" + index.getName(); - writer.earlyOpen(dir + "/", + if (!writer.open(dir + "/", 64, 262144, _dynamicKPosIndexFormat, index.getSchema(), index.getIndex(), - _tuneFileIndexing._write); - // No checkpointing - if (!writer.lateOpen(_tuneFileIndexing._write, - _fileHeaderContext)) { + _tuneFileIndexing._write, + _fileHeaderContext)) { LOG(error, "Could not open output posocc + dictionary in %s", dir.c_str()); abort(); diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp index aacfa400755..171e862f064 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp @@ -292,13 +292,9 @@ FileHandle::open(const vespalib::stringref &dir, _fieldWriter = new FieldWriter(docIdLimit, numWordIds); - _fieldWriter->earlyOpen(dir + "/", 64, 262144u, false, - index.getSchema(), index.getIndex(), - tuneFileWrite); - - // No checkpointing - - if (!_fieldWriter->lateOpen(tuneFileWrite, fileHeaderContext)) { + if (!_fieldWriter->open(dir + "/", 64, 262144u, false, + index.getSchema(), index.getIndex(), + tuneFileWrite, fileHeaderContext)) { LOG(error, "Could not open term writer %s for write (%s)", dir.c_str(), getLastErrorString().c_str()); abort(); diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp index 9c63c5930b6..72fd07e5752 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp @@ -4,7 +4,6 @@ #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/log/log.h> @@ -63,8 +62,7 @@ PageDict4FileSeqRead::PageDict4FileSeqRead() _ssCompleted(false), _spCompleted(false), _pCompleted(false), - _wordNum(0u), - _checkPointData() + _wordNum(0u) { _ssd.setReadContext(&_ssReadContext); _spd.setReadContext(&_spReadContext); @@ -229,26 +227,13 @@ PageDict4FileSeqRead::open(const vespalib::string &name, 32768u); _ssd.emptyBuffer(0); - if (_checkPointData) { - _ssReadContext.setPosition(_ssReadContext.getCheckPointOffset()); - if (_ssd._valI >= _ssd._valE) - _ssReadContext.readComprBuffer(); - _spReadContext.setPosition(_spReadContext.getCheckPointOffset()); - if (_spd._valI >= _spd._valE) - _spReadContext.readComprBuffer(); - _pReadContext.setPosition(_pReadContext.getCheckPointOffset()); - if (_pd._valI >= _pd._valE) - _pReadContext.readComprBuffer(); - } else { - _ssReadContext.readComprBuffer(); - assert(_ssReadContext.getBufferEndFilePos() >= fileSize); - readSSHeader(); - _spReadContext.readComprBuffer(); - readSPHeader(); - _pReadContext.readComprBuffer(); - readPHeader(); - } - + _ssReadContext.readComprBuffer(); + assert(_ssReadContext.getBufferEndFilePos() >= fileSize); + readSSHeader(); + _spReadContext.readComprBuffer(); + readSPHeader(); + _pReadContext.readComprBuffer(); + readPHeader(); _ssReader = new SSReader(_ssReadContext, _ssHeaderLen, @@ -263,15 +248,9 @@ PageDict4FileSeqRead::open(const vespalib::string &name, _spd, _pd); - if (_checkPointData) { - _ssReader->checkPointRead(*_checkPointData); - _pReader->checkPointRead(*_checkPointData); - assert(_checkPointData->empty()); - } else { - _ssReader->setup(_ssd); - _pReader->setup(); - _wordNum = 0; - } + _ssReader->setup(_ssd); + _pReader->setup(); + _wordNum = 0; return true; } @@ -299,40 +278,6 @@ PageDict4FileSeqRead::close() void -PageDict4FileSeqRead::checkPointWrite(vespalib::nbostream &out) -{ - _ssd.checkPointWrite(out); - _spReadContext.checkPointWrite(out); - _pReadContext.checkPointWrite(out); - vespalib::nbostream data; - _ssReader->checkPointWrite(data); - _pReader->checkPointWrite(data); - std::vector<char> checkPointData(data.size()); - data.read(&checkPointData[0], data.size()); - out << checkPointData; - out << _wordNum; - out << _ssCompleted << _ssFileBitSize << _ssHeaderLen; - out << _spCompleted << _spFileBitSize << _spHeaderLen; - out << _pCompleted << _pFileBitSize << _pHeaderLen; -} - -void -PageDict4FileSeqRead::checkPointRead(vespalib::nbostream &in) -{ - _ssd.checkPointRead(in); - _spReadContext.checkPointRead(in); - _pReadContext.checkPointRead(in); - std::vector<char> checkPointData; - in >> checkPointData; - _checkPointData.reset(new vespalib::nbostream(checkPointData.size())); - _checkPointData->write(&checkPointData[0], checkPointData.size()); - in >> _wordNum; - in >> _ssCompleted >> _ssFileBitSize >> _ssHeaderLen; - in >> _spCompleted >> _spFileBitSize >> _spHeaderLen; - in >> _pCompleted >> _pFileBitSize >> _pHeaderLen; -} - -void PageDict4FileSeqRead::getParams(PostingListParams ¶ms) { params.clear(); @@ -418,11 +363,9 @@ PageDict4FileSeqWrite::open(const vespalib::string &name, assertOpenWriteOnly(ok, ssname); _ssWriteContext.setFile(&_ssfile); - if (!_checkPointData) { - _pWriteContext.allocComprBuf(65536u, 32768u); - _spWriteContext.allocComprBuf(65536u, 32768u); - _ssWriteContext.allocComprBuf(65536u, 32768u); - } + _pWriteContext.allocComprBuf(65536u, 32768u); + _spWriteContext.allocComprBuf(65536u, 32768u); + _ssWriteContext.allocComprBuf(65536u, 32768u); uint64_t pFileSize = _pfile.GetSize(); uint64_t spFileSize = _spfile.GetSize(); @@ -445,33 +388,24 @@ PageDict4FileSeqWrite::open(const vespalib::string &name, assert(ssBufferStartFilePos == static_cast<uint64_t>(_ssfile.GetPosition())); - if (!_checkPointData) { - _pe.setupWrite(_pWriteContext); - _spe.setupWrite(_spWriteContext); - _sse.setupWrite(_ssWriteContext); - assert(_pe.getWriteOffset() == 0); - assert(_spe.getWriteOffset() == 0); - assert(_sse.getWriteOffset() == 0); - _spe.copyParams(_sse); - _pe.copyParams(_sse); - // Write initial file headers - makePHeader(fileHeaderContext); - makeSPHeader(fileHeaderContext); - makeSSHeader(fileHeaderContext); - } + _pe.setupWrite(_pWriteContext); + _spe.setupWrite(_spWriteContext); + _sse.setupWrite(_ssWriteContext); + assert(_pe.getWriteOffset() == 0); + assert(_spe.getWriteOffset() == 0); + assert(_sse.getWriteOffset() == 0); + _spe.copyParams(_sse); + _pe.copyParams(_sse); + // Write initial file headers + makePHeader(fileHeaderContext); + makeSPHeader(fileHeaderContext); + makeSSHeader(fileHeaderContext); _ssWriter = new SSWriter(_sse); _spWriter = new SPWriter(*_ssWriter, _spe); _pWriter = new PWriter(*_spWriter, _pe); - if (_checkPointData) { - _ssWriter->checkPointRead(*_checkPointData); - _spWriter->checkPointRead(*_checkPointData); - _pWriter->checkPointRead(*_checkPointData); - assert(_checkPointData->empty()); - } else { - _spWriter->setup(); - _pWriter->setup(); - } + _spWriter->setup(); + _pWriter->setup(); return true; } @@ -683,43 +617,6 @@ PageDict4FileSeqWrite::updateSSHeader(uint64_t fileBitSize) void -PageDict4FileSeqWrite::checkPointWrite(vespalib::nbostream &out) -{ - _ssWriteContext.writeComprBuffer(true); - _spWriteContext.writeComprBuffer(true); - _pWriteContext.writeComprBuffer(true); - _ssWriteContext.checkPointWrite(out); - _spWriteContext.checkPointWrite(out); - _pWriteContext.checkPointWrite(out); - vespalib::nbostream data; - _ssWriter->checkPointWrite(data); - _spWriter->checkPointWrite(data); - _pWriter->checkPointWrite(data); - std::vector<char> checkPointData(data.size()); - data.read(&checkPointData[0], data.size()); - out << checkPointData; - out << _ssHeaderLen << _spHeaderLen << _pHeaderLen; - _ssfile.Sync(); - _spfile.Sync(); - _pfile.Sync(); -} - - -void -PageDict4FileSeqWrite::checkPointRead(vespalib::nbostream &in) -{ - _ssWriteContext.checkPointRead(in); - _spWriteContext.checkPointRead(in); - _pWriteContext.checkPointRead(in); - std::vector<char> checkPointData; - in >> checkPointData; - _checkPointData.reset(new vespalib::nbostream(checkPointData.size())); - _checkPointData->write(&checkPointData[0], checkPointData.size()); - in >> _ssHeaderLen >> _spHeaderLen >> _pHeaderLen; -} - - -void PageDict4FileSeqWrite::setParams(const PostingListParams ¶ms) { params.get("avgBitsPerDoc", _sse._avgBitsPerDoc); diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h index fe432a22463..7baa883a740 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h @@ -50,8 +50,6 @@ class PageDict4FileSeqRead : public index::DictionaryFileSeqRead uint64_t _wordNum; - std::unique_ptr<vespalib::nbostream> _checkPointData; - void readSSHeader(); @@ -85,21 +83,6 @@ public: */ virtual bool close() override; - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void - checkPointWrite(vespalib::nbostream &out) override; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void - checkPointRead(vespalib::nbostream &in) override; - /* * Get current parameters. */ @@ -142,8 +125,6 @@ class PageDict4FileSeqWrite : public index::DictionaryFileSeqWrite uint32_t _spHeaderLen; // Length of header for sparse page file (bytes) uint32_t _ssHeaderLen; // Length of header for sparse sparse file (bytes) - std::unique_ptr<vespalib::nbostream> _checkPointData; - void writeIndexNames(vespalib::GenericHeader &header); @@ -196,21 +177,6 @@ public: virtual bool close() override; - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void - checkPointWrite(vespalib::nbostream &out) override; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void - checkPointRead(vespalib::nbostream &in) override; - /* * Set parameters. */ diff --git a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp index 00b6d23fbcf..3bd879de291 100644 --- a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "wordnummapper.h" -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/fastlib/io/bufferedfile.h> #include <cassert> diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index 768ee972097..17d4457a318 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -6,7 +6,6 @@ #include <vespa/searchlib/index/postinglistfile.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/common/fileheadercontext.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/log/log.h> @@ -27,7 +26,6 @@ using index::PostingListCountFileSeqWrite; using common::FileHeaderContext; using bitcompression::FeatureDecodeContextBE; using bitcompression::FeatureEncodeContextBE; -using vespalib::nbostream; using vespalib::getLastErrorString; @@ -78,11 +76,7 @@ Zc4PostingSeqRead(PostingListCountFileSeqRead *countFile) _rangeEndOffset(0), _readAheadEndOffset(0), _wordStart(0), - _checkPointPos(0), - _residue(0), - _checkPointChunkNo(0u), - _checkPointResidue(0u), - _checkPointHasMore(false) + _residue(0) { if (_countFile != NULL) { PostingListParams params; @@ -231,32 +225,6 @@ readDocIdAndFeatures(DocIdAndFeatures &features) void -Zc4PostingSeqRead::checkPointWrite(nbostream &out) -{ - out << _counts; - out << _wordStart; - uint64_t curPos = _decodeContext->getReadOffset(); - out << curPos; - out << _residue; - out << _chunkNo; - out << _hasMore; -} - - -void -Zc4PostingSeqRead::checkPointRead(nbostream &in) -{ - in >> _counts; - in >> _wordStart; - in >> _checkPointPos; - in >> _checkPointResidue; - in >> _checkPointChunkNo; - in >> _checkPointHasMore; - assert(_checkPointPos >= _wordStart); -} - - -void Zc4PostingSeqRead::readWordStartWithSkip() { typedef FeatureEncodeContextBE EC; @@ -508,34 +476,8 @@ Zc4PostingSeqRead::open(const vespalib::string &name, _readContext.readComprBuffer(); readHeader(); - if (d._valI >= d._valE) + if (d._valI >= d._valE) { _readContext.readComprBuffer(); - if (_checkPointPos != 0) { - if (_checkPointResidue != 0 || _checkPointHasMore) { - // Checkpointed in the middle of a word. Read from - // start at word until at right position. - DocIdAndFeatures features; - _readContext.setPosition(_wordStart); - assert(_decodeContext->getReadOffset() == _wordStart); - _readContext.readComprBuffer(); - readWordStart(); - assert(_chunkNo < _checkPointChunkNo || - (_chunkNo == _checkPointChunkNo && - _residue >= _checkPointResidue)); - while (_chunkNo < _checkPointChunkNo || - _residue > _checkPointResidue) { - readDocIdAndFeatures(features); - } - assert(_chunkNo == _checkPointChunkNo); - assert(_residue == _checkPointResidue); - assert(_hasMore == _checkPointHasMore); - assert(_decodeContext->getReadOffset() == _checkPointPos); - } else { - // Checkpointed between words. - _readContext.setPosition(_checkPointPos); - assert(_decodeContext->getReadOffset() == _checkPointPos); - _readContext.readComprBuffer(); - } } } else { LOG(error, "could not open %s: %s", @@ -740,29 +682,6 @@ Zc4PostingSeqWrite::flushWord() } -void -Zc4PostingSeqWrite::checkPointWrite(nbostream &out) -{ - _writeContext.writeComprBuffer(true); // Also flush slack - out << _numWords; - _writeContext.checkPointWrite(out); - _featureWriteContext.checkPointWrite(out); - out.saveVector(_docIds) << _writePos << _counts; - _file.Sync(); -} - - -void -Zc4PostingSeqWrite::checkPointRead(nbostream &in) -{ - in >> _numWords; - _writeContext.checkPointRead(in); - _featureWriteContext.checkPointRead(in); - _featureOffset = _encodeFeatures->getWriteOffset(); - in.restoreVector(_docIds) >> _writePos >> _counts; -} - - uint32_t Zc4PostingSeqWrite::readHeader(const vespalib::string &name) { @@ -1422,10 +1341,3 @@ ZcPostingSeqWrite::flushWordNoSkip() } } // namespace search::diskindex - -#include <vespa/vespalib/objects/nbostream.hpp> -namespace vespalib { - using UInt32Pair = std::pair<uint32_t, uint32_t>; - template nbostream &nbostream::saveVector<UInt32Pair>(const std::vector<UInt32Pair> &); - template nbostream &nbostream::restoreVector<UInt32Pair>(std::vector<UInt32Pair> &); -} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.h b/searchlib/src/vespa/searchlib/diskindex/zcposting.h index 64879aa05fb..9a650abdd40 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.h @@ -78,11 +78,7 @@ protected: uint64_t _rangeEndOffset; // End offset for word pair uint64_t _readAheadEndOffset; // Readahead end offset for word pair uint64_t _wordStart; // last word header position - uint64_t _checkPointPos; // file position when checkpointing uint32_t _residue; // Number of unread documents after word header - uint32_t _checkPointChunkNo; // _chunkNo when checkpointing - uint32_t _checkPointResidue; // _residue when checkpointing - bool _checkPointHasMore; // _hasMore when checkpointing public: Zc4PostingSeqRead(index::PostingListCountFileSeqRead *countFile); @@ -98,8 +94,6 @@ public: virtual void readCommonWordDocIdAndFeatures(DocIdAndFeatures &features); void readDocIdAndFeatures(DocIdAndFeatures &features) override; - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; void readCounts(const PostingListCounts &counts) override; // Fill in for next word bool open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) override; bool close() override; @@ -176,8 +170,6 @@ public: void writeDocIdAndFeatures(const DocIdAndFeatures &features) override; void flushWord() override; - void checkPointWrite(vespalib::nbostream &out) override; - void checkPointRead(vespalib::nbostream &in) override; bool open(const vespalib::string &name, const TuneFileSeqWrite &tuneFileWrite, const search::common::FileHeaderContext &fileHeaderContext) override; diff --git a/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp b/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp index 6c14498be76..513c542637d 100644 --- a/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp +++ b/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp @@ -1,14 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docidandfeatures.h" -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".index.docidandfeatures"); namespace search::index { -using vespalib::nbostream; - DocIdAndFeatures::DocIdAndFeatures() : _docId(0), _wordDocFeatures(), @@ -24,100 +21,4 @@ DocIdAndFeatures::DocIdAndFeatures(const DocIdAndFeatures &) = default; DocIdAndFeatures & DocIdAndFeatures::operator = (const DocIdAndFeatures &) = default; DocIdAndFeatures::~DocIdAndFeatures() { } -#if 0 -void -DocIdAndFeatures::append(const DocIdAndFeatures &rhs, uint32_t localFieldId) -{ - assert(!rhs.getRaw()); - assert(rhs._fields.size() == 1); - const WordDocFieldFeatures &field = rhs._fields.front(); - assert(field.getFieldId() == 0); - uint32_t numElements = field.getNumElements(); - std::vector<WordDocFieldElementFeatures>::const_iterator element = - rhs._elements.begin(); - std::vector<WordDocFieldElementWordPosFeatures>::const_iterator position = - rhs._wordPositions.begin(); - assert(_fields.empty() || localFieldId > _fields.back().getFieldId()); - _fields.push_back(field); - _fields.back().setFieldId(localFieldId); - for (uint32_t elementDone = 0; elementDone < numElements; - ++elementDone, ++element) { - _elements.push_back(*element); - for (uint32_t posResidue = element->getNumOccs(); posResidue > 0; - --posResidue, ++position) { - _wordPositions.push_back(*position); - } - } -} -#endif - - -nbostream & -operator<<(nbostream &out, const WordDocElementFeatures &features) -{ - out << features._elementId << features._numOccs << - features._weight << features._elementLen; - return out; -} - - -nbostream & -operator>>(nbostream &in, WordDocElementFeatures &features) -{ - in >> features._elementId >> features._numOccs >> - features._weight >> features._elementLen; - return in; -} - - -nbostream & -operator<<(nbostream &out, const WordDocElementWordPosFeatures &features) -{ - out << features._wordPos; - return out; -} - - -nbostream & -operator>>(nbostream &in, WordDocElementWordPosFeatures &features) -{ - in >> features._wordPos; - return in; -} - - -nbostream & -operator<<(nbostream &out, const DocIdAndFeatures &features) -{ - out << features._docId; - out.saveVector(features._elements). - saveVector(features._wordPositions); - out.saveVector(features._blob); - out << features._bitOffset << features._bitLength << features._raw; - return out; -} - - -nbostream & -operator>>(nbostream &in, DocIdAndFeatures &features) -{ - in >> features._docId; - in.restoreVector(features._elements). - restoreVector(features._wordPositions); - in.restoreVector(features._blob); - in >> features._bitOffset >> features._bitLength >> features._raw; - return in; -} - -} - -#include <vespa/vespalib/objects/nbostream.hpp> - -namespace vespalib { - using search::index::WordDocElementFeatures; - using search::index::WordDocElementWordPosFeatures; - template nbostream& nbostream::saveVector<WordDocElementFeatures>(const std::vector<WordDocElementFeatures> &); - template nbostream& nbostream::restoreVector<WordDocElementFeatures>(std::vector<WordDocElementFeatures> &); - template nbostream& nbostream::saveVector<WordDocElementWordPosFeatures>(const std::vector<WordDocElementWordPosFeatures> &); - template nbostream& nbostream::restoreVector<WordDocElementWordPosFeatures>(std::vector<WordDocElementWordPosFeatures> &); } diff --git a/searchlib/src/vespa/searchlib/index/docidandfeatures.h b/searchlib/src/vespa/searchlib/index/docidandfeatures.h index 5b8ded5f1e0..91a500495cc 100644 --- a/searchlib/src/vespa/searchlib/index/docidandfeatures.h +++ b/searchlib/src/vespa/searchlib/index/docidandfeatures.h @@ -5,8 +5,6 @@ #include <vector> #include <cstdint> -namespace vespalib { class nbostream; } - namespace search::index { /* @@ -195,24 +193,4 @@ public: void append(const DocIdAndFeatures &rhs, uint32_t localFieldId); }; - -vespalib::nbostream & -operator<<(vespalib::nbostream &out, const WordDocElementFeatures &features); - -vespalib::nbostream & -operator>>(vespalib::nbostream &in, WordDocElementFeatures &features); - -vespalib::nbostream & -operator<<(vespalib::nbostream &out, const WordDocElementWordPosFeatures &features); - -vespalib::nbostream & -operator>>(vespalib::nbostream &in, WordDocElementWordPosFeatures &features); - -vespalib::nbostream & -operator<<(vespalib::nbostream &out, const DocIdAndFeatures &features); - -vespalib::nbostream & -operator>>(vespalib::nbostream &in, DocIdAndFeatures &features); - } - diff --git a/searchlib/src/vespa/searchlib/index/postinglistcountfile.h b/searchlib/src/vespa/searchlib/index/postinglistcountfile.h index 5b14b477c42..30c6963f228 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistcountfile.h +++ b/searchlib/src/vespa/searchlib/index/postinglistcountfile.h @@ -5,8 +5,6 @@ #include "postinglistcounts.h" #include <vespa/searchlib/common/tunefileinfo.h> -namespace vespalib { class nbostream; } - namespace search { namespace common { class FileHeaderContext; } @@ -34,19 +32,6 @@ public: virtual ~PostingListCountFileSeqRead(); /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; - - /** * Open posting list count file for sequential read. */ virtual bool open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) = 0; @@ -71,19 +56,6 @@ public: virtual ~PostingListCountFileSeqWrite(); /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; - - /** * Open posting list count file for sequential write. */ virtual bool open(const vespalib::string &name, diff --git a/searchlib/src/vespa/searchlib/index/postinglistcounts.cpp b/searchlib/src/vespa/searchlib/index/postinglistcounts.cpp index d7bfcca03cc..23b0bf6b6a3 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistcounts.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglistcounts.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/objects/nbostream.h> #include "postinglistcounts.h" namespace search::index { @@ -10,71 +9,4 @@ void swap(PostingListCounts & a, PostingListCounts & b) a.swap(b); } -using vespalib::nbostream; - -nbostream & -operator<<(nbostream &out, const PostingListCounts::Segment &segment) -{ - out << segment._bitLength << segment._numDocs << segment._lastDoc; - return out; -} - - -nbostream & -operator>>(nbostream &in, PostingListCounts::Segment &segment) -{ - in >> segment._bitLength >> segment._numDocs >> segment._lastDoc; - return in; -} - - -nbostream & -operator<<(nbostream &out, const PostingListCounts &counts) -{ - out << counts._numDocs << counts._bitLength; - size_t numSegments = counts._segments.size(); - out << numSegments; - for (size_t seg = 0; seg < numSegments; ++seg) { - out << counts._segments[seg]; - } - return out; -} - - -nbostream & -operator>>(nbostream &in, PostingListCounts &counts) -{ - in >> counts._numDocs >> counts._bitLength; - size_t numSegments = 0; - in >> numSegments; - counts._segments.reserve(numSegments); - counts._segments.clear(); - for (size_t seg = 0; seg < numSegments; ++seg) { - PostingListCounts::Segment segment; - in >> segment; - counts._segments.push_back(segment); - } - return in; -} - - -nbostream & -operator<<(nbostream &out, const PostingListOffsetAndCounts &offsetAndCounts) -{ - out << offsetAndCounts._offset; - out << offsetAndCounts._accNumDocs; - out << offsetAndCounts._counts; - return out; -} - - -nbostream & -operator>>(nbostream &in, PostingListOffsetAndCounts &offsetAndCounts) -{ - in >> offsetAndCounts._offset; - in >> offsetAndCounts._accNumDocs; - in >> offsetAndCounts._counts; - return in; -} - } diff --git a/searchlib/src/vespa/searchlib/index/postinglistcounts.h b/searchlib/src/vespa/searchlib/index/postinglistcounts.h index f59b906e982..d11f7286fa1 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistcounts.h +++ b/searchlib/src/vespa/searchlib/index/postinglistcounts.h @@ -4,8 +4,6 @@ #include <vector> #include <cstdint> -namespace vespalib { class nbostream; } - namespace search { namespace index { @@ -47,12 +45,6 @@ public: _numDocs == rhs._numDocs && _lastDoc == rhs._lastDoc); } - - friend vespalib::nbostream & - operator<<(vespalib::nbostream &out, const Segment &segment); - - friend vespalib::nbostream & - operator>>(vespalib::nbostream &in, Segment &segment); }; /** @@ -92,12 +84,6 @@ public: _bitLength == rhs._bitLength && _segments == rhs._segments); } - - friend vespalib::nbostream & - operator<<(vespalib::nbostream &out, const PostingListCounts &counts); - - friend vespalib::nbostream & - operator>>(vespalib::nbostream &in, PostingListCounts &counts); }; void swap(PostingListCounts & a, PostingListCounts & b); @@ -115,12 +101,6 @@ public: _accNumDocs(0u), _counts() { } - - friend vespalib::nbostream & - operator<<(vespalib::nbostream &out, const PostingListOffsetAndCounts &offsetAndCounts); - - friend vespalib::nbostream & - operator>>(vespalib::nbostream &in, PostingListOffsetAndCounts &offsetAndCounts); }; } // namespace index diff --git a/searchlib/src/vespa/searchlib/index/postinglistfile.h b/searchlib/src/vespa/searchlib/index/postinglistfile.h index 43628bd798b..61247ef3197 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistfile.h +++ b/searchlib/src/vespa/searchlib/index/postinglistfile.h @@ -8,8 +8,6 @@ class FastOS_FileInterface; -namespace vespalib { class nbostream; } - namespace search { namespace common { class FileHeaderContext; } @@ -38,19 +36,6 @@ public: virtual void readDocIdAndFeatures(DocIdAndFeatures &features) = 0; /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; - - /** * Read counts for a word. */ virtual void readCounts(const PostingListCounts &counts) = 0; @@ -135,19 +120,6 @@ public: virtual void flushWord() = 0; /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Implies - * flush from memory to disk, and possibly also sync to permanent - * storage media. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; - - /** * Open posting list file for sequential write. */ virtual bool diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp index bfc7f7032c1..1fa518af28f 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp @@ -553,13 +553,10 @@ FakeWord::validate(search::queryeval::SearchIterator *iterator, bool verbose) co bool -FakeWord::validate(std::shared_ptr<FieldReader> &fieldReader, +FakeWord::validate(FieldReader &fieldReader, uint32_t wordNum, const fef::TermFieldMatchDataArray &matchData, - bool verbose, - uint32_t &checkPointCheck, - uint32_t checkPointInterval, - CheckPointCallback *const checkPointCallback) const + bool verbose) const { uint32_t docId = 0; uint32_t numDocs; @@ -585,8 +582,8 @@ FakeWord::validate(std::shared_ptr<FieldReader> &fieldReader, #endif numDocs = _postings.size(); for (residue = numDocs; residue > 0; --residue) { - assert(fieldReader->_wordNum == wordNum); - DocIdAndFeatures &features(fieldReader->_docIdAndFeatures); + assert(fieldReader._wordNum == wordNum); + DocIdAndFeatures &features(fieldReader._docIdAndFeatures); docId = features._docId; assert(d != de); assert(d->_docId == docId); @@ -648,12 +645,7 @@ FakeWord::validate(std::shared_ptr<FieldReader> &fieldReader, assert(presidue == 0); ++d; } - if (++checkPointCheck >= checkPointInterval) { - checkPointCheck = 0; - if (checkPointCallback != NULL) - checkPointCallback->checkPoint(); - } - fieldReader->read(); + fieldReader.read(); } if (matchData.valid()) { assert(p == pe); @@ -707,11 +699,8 @@ FakeWord::validate(const search::BitVector &bv) const bool -FakeWord::dump(std::shared_ptr<FieldWriter> &fieldWriter, - bool verbose, - uint32_t &checkPointCheck, - uint32_t checkPointInterval, - CheckPointCallback *checkPointCallback) const +FakeWord::dump(FieldWriter &fieldWriter, + bool verbose) const { uint32_t numDocs; uint32_t residue; @@ -732,13 +721,8 @@ FakeWord::dump(std::shared_ptr<FieldWriter> &fieldWriter, assert(d != de); setupFeatures(*d, &*p, features); p += d->_positions; - fieldWriter->add(features); + fieldWriter.add(features); ++d; - if (++checkPointCheck >= checkPointInterval) { - checkPointCheck = 0; - if (checkPointCallback != NULL) - checkPointCallback->checkPoint(); - } } assert(p == pe); assert(d == de); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h index 9417c32f1e5..619c1760797 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h @@ -17,15 +17,6 @@ namespace search { namespace fakedata { -class CheckPointCallback -{ -public: - CheckPointCallback() {} - virtual ~CheckPointCallback() {} - - virtual void checkPoint() = 0; -}; - /* * General representation of a faked word, containing all features used * by any of the candidate posting list formats. @@ -254,23 +245,17 @@ public: bool validate(search::queryeval::SearchIterator *iterator, bool verbose) const; bool - validate(std::shared_ptr<search::diskindex::FieldReader> &fieldReader, + validate(search::diskindex::FieldReader &fieldReader, uint32_t wordNum, const fef::TermFieldMatchDataArray &matchData, - bool verbose, - uint32_t &checkPointCheck, - uint32_t checkPointInterval, - CheckPointCallback *const checkPointCallback) const; + bool verbose) const; void validate(const std::vector<uint32_t> &docIds) const; void validate(const BitVector &bv) const; bool - dump(std::shared_ptr<search::diskindex::FieldWriter> &fieldWriter, - bool verbose, - uint32_t &checkPointCheck, - uint32_t checkPointInterval, - CheckPointCallback *checkPointCallback) const; + dump(search::diskindex::FieldWriter &fieldWriter, + bool verbose) const; const std::string &getName() const { return _name; } uint32_t getDocIdLimit() const { return _docIdLimit; } diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp index 8c7f97db022..acfc2e46020 100644 --- a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp +++ b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp @@ -1,14 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "comprbuffer.h" -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/fastos/file.h> #include <cassert> +#include <cstring> namespace search { -using vespalib::nbostream; - ComprBuffer::ComprBuffer(uint32_t unitSize) : _comprBuf(NULL), _comprBufSize(0), @@ -114,25 +112,4 @@ ComprBuffer::referenceComprBuf(const ComprBuffer &rhs) _comprBufSize = rhs._comprBufSize; } - -void -ComprBuffer::checkPointWrite(nbostream &out) -{ - _aligner.checkPointWrite(out); - out << _comprBufSize << _unitSize << _padBefore; -} - - -void -ComprBuffer::checkPointRead(nbostream &in) -{ - _aligner.checkPointRead(in); - uint32_t unitSize; - in >> _comprBufSize >> unitSize >> _padBefore; - assert(unitSize == _unitSize); - - allocComprBuf(); -} - - } diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.h b/searchlib/src/vespa/searchlib/util/comprbuffer.h index 61928976acc..c15671d5e14 100644 --- a/searchlib/src/vespa/searchlib/util/comprbuffer.h +++ b/searchlib/src/vespa/searchlib/util/comprbuffer.h @@ -31,7 +31,6 @@ public: static size_t minimumPadding() { return 8; } uint32_t getUnitBitSize() const { return _unitSize * 8; } bool getPadBefore() const { return _padBefore; } - bool getCheckPointResumed() const { return _aligner.getCheckPointResumed(); } /* * When encoding to memory instead of file, the compressed buffer must @@ -44,18 +43,6 @@ public: * long as rhs is live and unchanged. */ void referenceComprBuf(const ComprBuffer &rhs); - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. - */ - void checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - * - */ - void checkPointRead(vespalib::nbostream &in); }; } diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp index 8b17fd85753..26c66f43993 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.cpp +++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp @@ -1,14 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "comprfile.h" -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/fastos/file.h> #include <cassert> +#include <cstring> namespace search { -using vespalib::nbostream; - void ComprFileReadBase::ReadComprBuffer(uint64_t stopOffset, bool readAll, @@ -292,9 +290,7 @@ ComprFileReadContext(ComprFileDecodeContext &decodeContext) _bitOffset(0), _stopOffset(0), _readAll(true), - _checkPointOffsetValid(false), - _file(NULL), - _checkPointOffset(0) + _file(NULL) { } @@ -308,9 +304,7 @@ ComprFileReadContext(uint32_t unitSize) _bitOffset(0), _stopOffset(0), _readAll(true), - _checkPointOffsetValid(false), - _file(NULL), - _checkPointOffset(0) + _file(NULL) { } @@ -497,28 +491,6 @@ ComprFileReadContext::copyReadContext(const ComprFileReadContext &rhs) } } - -void -ComprFileReadContext::checkPointWrite(nbostream &out) -{ - ComprBuffer::checkPointWrite(out); - ComprFileDecodeContext &d = *_decodeContext; - d.checkPointWrite(out); - uint64_t bitOffset = d.getBitPosV(); - out << bitOffset; -} - - -void -ComprFileReadContext::checkPointRead(nbostream &in) -{ - ComprBuffer::checkPointRead(in); - ComprFileDecodeContext &d = *_decodeContext; - d.checkPointRead(in); - in >> _checkPointOffset; // Cannot seek until file is opened - _checkPointOffsetValid = true; -} - ComprFileWriteContext:: ComprFileWriteContext(ComprFileEncodeContext &encodeContext) : ComprBuffer(encodeContext.getUnitByteSize()), @@ -598,47 +570,4 @@ ComprFileWriteContext::allocComprBuf() allocComprBuf(32768, 32768); } - -void -ComprFileWriteContext::checkPointWrite(nbostream &out) -{ - ComprBuffer::checkPointWrite(out); - ComprFileEncodeContext &e = *_encodeContext; - uint64_t bufferStartFilePos = getBufferStartFilePos(); - uint64_t usedSize = e.getUsedUnits(_comprBuf) * - e.getUnitByteSize(); - out << bufferStartFilePos << usedSize; - e.checkPointWrite(out); - if (usedSize != 0) { - out.write(_comprBuf, usedSize); - } - uint64_t bitOffset = e.getBitPosV(); - out << bitOffset; -} - - -void -ComprFileWriteContext::checkPointRead(nbostream &in) -{ - ComprBuffer::checkPointRead(in); - ComprFileEncodeContext &e = *_encodeContext; - uint64_t bufferStartFilePos = 0; - uint64_t usedSize = 0; - in >> bufferStartFilePos >> usedSize; - e.checkPointRead(in); - if (usedSize != 0) { - assert((usedSize % e.getUnitByteSize()) == 0); - assert(_comprBufSize >= usedSize / e.getUnitByteSize()); - in.read(_comprBuf, usedSize); - } - setBufferStartFilePos(bufferStartFilePos); - e.afterWrite(*this, usedSize / e.getUnitByteSize(), bufferStartFilePos); - uint64_t bitOffset = 0; - in >> bitOffset; - uint64_t writeOffset = e.getBitPosV(); - assert(bitOffset == writeOffset); - (void) writeOffset; -} - - } diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h index b69ef140868..3d44f088c74 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.h +++ b/searchlib/src/vespa/searchlib/util/comprfile.h @@ -7,8 +7,6 @@ class FastOS_FileInterface; -namespace vespalib { class nbostream; } - namespace search { class ComprFileWriteContext; @@ -60,19 +58,6 @@ public: * Get size of each unit (typically 4 or 8) */ virtual uint32_t getUnitByteSize() const = 0; - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Caller must - * save position. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - * Caller must restore position. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; }; class ComprFileReadBase @@ -110,9 +95,7 @@ private: int _bitOffset; uint64_t _stopOffset; bool _readAll; - bool _checkPointOffsetValid; // Set only if checkpoint has been read FastOS_FileInterface *_file; - uint64_t _checkPointOffset; // bit offset saved by checkPointRead public: ComprFileReadContext(ComprFileDecodeContext &decodeContext); @@ -170,19 +153,6 @@ public: * For unit testing only. Copy data owned by rhs. */ void copyWriteContext(const ComprFileWriteContext &rhs); - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. - */ - void checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void checkPointRead(vespalib::nbostream &in); - bool getCheckPointOffsetValid() const { return _checkPointOffsetValid; } - uint64_t getCheckPointOffset() const { return _checkPointOffset; } }; @@ -216,19 +186,6 @@ public: */ virtual uint32_t getUnitByteSize() const = 0; - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. Caller must - * save position, although partial unit is saved. - */ - virtual void checkPointWrite(vespalib::nbostream &out) = 0; - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - * Caller must restore positon, although partial unit is restored. - */ - virtual void checkPointRead(vespalib::nbostream &in) = 0; - virtual uint64_t getBitPosV() const = 0; }; @@ -281,17 +238,6 @@ public: * no file is attached. */ std::pair<void *, size_t> grabComprBuffer(void *&comprBufMalloc); - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. - */ - void checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void checkPointRead(vespalib::nbostream &in); }; } diff --git a/searchlib/src/vespa/searchlib/util/filealign.cpp b/searchlib/src/vespa/searchlib/util/filealign.cpp index 15bef611714..b60c0492854 100644 --- a/searchlib/src/vespa/searchlib/util/filealign.cpp +++ b/searchlib/src/vespa/searchlib/util/filealign.cpp @@ -1,14 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "filealign.h" -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/fastos/file.h> #include <cassert> namespace search { -using vespalib::nbostream; - namespace { size_t @@ -42,8 +39,7 @@ FileAlign::FileAlign() _minAlignedSize(1), _elemSize(1), _directIOMemAlign(1), - _directio(false), - _checkPointResumed(false) + _directio(false) { } @@ -116,24 +112,4 @@ FileAlign::setupAlign(size_t elements, return elements; } - -void -FileAlign::checkPointWrite(nbostream &out) -{ - out << _directIOFileAlign << _preferredFileAlign << - _minDirectIOSize << _minAlignedSize << _elemSize << - _directIOMemAlign << _directio; -} - - -void -FileAlign::checkPointRead(nbostream &in) -{ - in >> _directIOFileAlign >> _preferredFileAlign >> - _minDirectIOSize >> _minAlignedSize >> _elemSize >> - _directIOMemAlign >> _directio; - _checkPointResumed = true; -} - - } diff --git a/searchlib/src/vespa/searchlib/util/filealign.h b/searchlib/src/vespa/searchlib/util/filealign.h index 3e75fb2f371..49d053957ce 100644 --- a/searchlib/src/vespa/searchlib/util/filealign.h +++ b/searchlib/src/vespa/searchlib/util/filealign.h @@ -7,8 +7,6 @@ class FastOS_FileInterface; -namespace vespalib { class nbostream; } - namespace search { class FileAlign @@ -21,7 +19,6 @@ private: size_t _elemSize; size_t _directIOMemAlign; bool _directio; - bool _checkPointResumed; public: @@ -64,24 +61,12 @@ public: */ size_t setupAlign(size_t elements, size_t elemSize, FastOS_FileInterface *file, size_t preferredFileAlignment); bool getDirectIO() const { return _directio; } - bool getCheckPointResumed() const { return _checkPointResumed; } size_t getDirectIOFileAlign() const { return _directIOFileAlign; } size_t getDirectIOMemAlign() const { return _directIOMemAlign; } size_t getMinDirectIOSize() const { return _minDirectIOSize; } size_t getMinAlignedSize() const { return _minAlignedSize; } size_t getPreferredFileAlign() const { return _preferredFileAlign; } size_t getElemSize() const { return _elemSize; } - - /** - * Checkpoint write. Used at semi-regular intervals during indexing - * to allow for continued indexing after an interrupt. - */ - void checkPointWrite(vespalib::nbostream &out); - - /** - * Checkpoint read. Used when resuming indexing after an interrupt. - */ - void checkPointRead(vespalib::nbostream &in); }; } diff --git a/vespalib/src/tests/objects/nbostream/nbostream_test.cpp b/vespalib/src/tests/objects/nbostream/nbostream_test.cpp index f2e24f5acdc..8b9ccb8a848 100644 --- a/vespalib/src/tests/objects/nbostream/nbostream_test.cpp +++ b/vespalib/src/tests/objects/nbostream/nbostream_test.cpp @@ -201,22 +201,6 @@ TEST_F("Test serializing std::pair", Fixture) f.assertSerialize(exp, val); } -TEST_F("Test saveVector", Fixture) -{ - std::vector<int16_t> val({ 0x0123, 0x4567 }); - val.reserve(16); - ExpBuffer exp({ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x01, 0x23, 0x45, 0x67 }); - f._stream.saveVector(val); - EXPECT_EQUAL(exp, f._stream); - std::vector<int16_t> checkVal; - f._stream.restoreVector(checkVal); - EXPECT_EQUAL(val, checkVal); - EXPECT_EQUAL(val.capacity(), checkVal.capacity()); -} - - TEST_F("Test write", Fixture) { f._stream.write("Hello", 5); diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.cpp b/vespalib/src/vespa/vespalib/objects/nbostream.cpp index d8e35a69d16..0225b788e68 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.cpp +++ b/vespalib/src/vespa/vespalib/objects/nbostream.cpp @@ -1,8 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "nbostream.hpp" +#include "nbostream.h" #include "hexdump.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> namespace vespalib { @@ -135,13 +136,4 @@ void nbostream::swap(nbostream & os) std::swap(_rbuf, os._rbuf); } -template nbostream& nbostream::saveVector<int16_t>(const std::vector<int16_t> &); -template nbostream& nbostream::restoreVector<int16_t>(std::vector<int16_t> &); -template nbostream& nbostream::saveVector<int32_t>(const std::vector<int32_t> &); -template nbostream& nbostream::restoreVector<int32_t>(std::vector<int32_t> &); -template nbostream& nbostream::saveVector<uint32_t>(const std::vector<uint32_t> &); -template nbostream& nbostream::restoreVector<uint32_t>(std::vector<uint32_t> &); -template nbostream& nbostream::saveVector<uint64_t>(const std::vector<uint64_t> &); -template nbostream& nbostream::restoreVector<uint64_t>(std::vector<uint64_t> &); - } diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.h b/vespalib/src/vespa/vespalib/objects/nbostream.h index 75d3bc03313..70a590f79d1 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.h +++ b/vespalib/src/vespa/vespalib/objects/nbostream.h @@ -138,16 +138,6 @@ public: return *this; } - // For checkpointing where capacity should be restored - template <typename T> - nbostream & - saveVector(const std::vector<T> &val); - - // For checkpointing where capacity should be restored - template <typename T> - nbostream & - restoreVector(std::vector<T> &val); - size_t size() const { return left(); } size_t capacity() const { return _wbuf.size(); } bool empty() const { return size() == 0; } diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.hpp b/vespalib/src/vespa/vespalib/objects/nbostream.hpp deleted file mode 100644 index 8b045f517a1..00000000000 --- a/vespalib/src/vespa/vespalib/objects/nbostream.hpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "nbostream.h" -#include <cassert> - -namespace vespalib { - -template <typename T> -nbostream & -nbostream::saveVector(const std::vector<T> &val) -{ - size_t valCapacity = val.capacity(); - size_t valSize = val.size(); - assert(valCapacity >= valSize); - *this << valCapacity << valSize; - for (const T & v : val) { - *this << v; - } - return *this; -} - -template <typename T> -nbostream & -nbostream::restoreVector(std::vector<T> &val) -{ - size_t valCapacity = 0; - size_t valSize = 0; - *this >> valCapacity >> valSize; - assert(valCapacity >= valSize); - val.reserve(valCapacity); - val.clear(); - T i; - for (size_t j = 0; j < valSize; ++j) { - *this >> i; - val.push_back(i); - } - return *this; -} - -} |