diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2017-10-20 22:19:40 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2018-01-10 09:56:23 +0100 |
commit | 2d8516e70764385df6e5bad05c319c27b7eb571a (patch) | |
tree | be779e2e9aafa78c2182f872fd26acf489ecb6ad /searchlib | |
parent | ebc215414e8ae36449f956f57e85843ee527e61a (diff) |
Combine crc and compression.
Diffstat (limited to 'searchlib')
10 files changed, 32 insertions, 29 deletions
diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index b389c1a8da5..9589d43c762 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -44,7 +44,7 @@ private: bool partialUpdateTest(); bool testVisitOverGeneratedDomain(); bool testRemove(); - void createAndFillDomain(const vespalib::string & name, DomainPart::Crc crcMethod, size_t preExistingDomains); + void createAndFillDomain(const vespalib::string & name, Encoding encoding, size_t preExistingDomains); void verifyDomain(const vespalib::string & name); void testCrcVersions(); bool testVisitOverPreExistingDomain(); @@ -491,10 +491,10 @@ bool Test::testVisitOverGeneratedDomain() return true; } -void Test::createAndFillDomain(const vespalib::string & name, DomainPart::Crc crcMethod, size_t preExistingDomains) +void Test::createAndFillDomain(const vespalib::string & name, Encoding encoding, size_t preExistingDomains) { DummyFileHeaderContext fileHeaderContext; - TransLogServer tlss("test13", 18377, ".", fileHeaderContext, 0x10000, 4, crcMethod); + TransLogServer tlss("test13", 18377, ".", fileHeaderContext, 0x10000, 4, encoding); TransLogClient tls("tcp/localhost:18377"); createDomainTest(tls, name, preExistingDomains); diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index 74efe3fe68e..9d5e080e138 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -21,4 +21,4 @@ usefsync bool default=false restart maxthreads int default=4 restart ##Default crc method used -crcmethod enum {ccitt_crc32, xxh64} default=xxh64 +crcmethod enum {ccitt_crc32, xxh64, xxh64lz4} default=xxh64 diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index ea8961dc564..c3b2738eac1 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -26,10 +26,10 @@ namespace search::transactionlog { Domain::Domain(const string &domainName, const string & baseDir, FastOS_ThreadPool & threadPool, Executor & commitExecutor, Executor & sessionExecutor, uint64_t domainPartSize, - DomainPart::Crc defaultCrcType, const FileHeaderContext &fileHeaderContext) : + Encoding defaultEncoding, const FileHeaderContext &fileHeaderContext) : _currentChunk(std::make_unique<Chunk>()), _lastSerial(0), - _defaultCrcType(defaultCrcType), + _defaultEncoding(defaultEncoding), _threadPool(threadPool), _commitExecutor(commitExecutor), _sessionExecutor(sessionExecutor), @@ -67,7 +67,7 @@ Domain::Domain(const string &domainName, const string & baseDir, FastOS_ThreadPo } _sessionExecutor.sync(); if (_parts.empty() || _parts.crbegin()->second->isClosed()) { - _parts[lastPart].reset(new DomainPart(_name, dir(), lastPart, _defaultCrcType, _fileHeaderContext, false)); + _parts[lastPart].reset(new DomainPart(_name, dir(), lastPart, _defaultEncoding, _fileHeaderContext, false)); } _lastSerial = end(); _self = _threadPool.NewThread(this); @@ -84,7 +84,7 @@ Domain::Run(FastOS_ThreadInterface *thisThread, void *) { } } void Domain::addPart(int64_t partId, bool isLastPart) { - DomainPart::SP dp(new DomainPart(_name, dir(), partId, _defaultCrcType, _fileHeaderContext, isLastPart)); + DomainPart::SP dp(new DomainPart(_name, dir(), partId, _defaultEncoding, _fileHeaderContext, isLastPart)); if (dp->size() == 0) { // Only last domain part is allowed to be truncated down to // empty size. @@ -381,7 +381,7 @@ Domain::commitChunk(std::unique_ptr<Chunk> chunk, const vespalib::MonitorGuard & triggerSyncNow(); waitPendingSync(_syncMonitor, _pendingSync); dp->close(); - dp.reset(new DomainPart(_name, dir(), entry.serial(), _defaultCrcType, _fileHeaderContext, false)); + dp.reset(new DomainPart(_name, dir(), entry.serial(), _defaultEncoding, _fileHeaderContext, false)); { LockGuard guard(_lock); _parts[entry.serial()] = dp; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index fb18bb955d7..c07459b0da9 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -42,7 +42,7 @@ public: using Executor = vespalib::ThreadExecutor; Domain(const vespalib::string &name, const vespalib::string &baseDir, FastOS_ThreadPool & threadPool, Executor & commitExecutor, Executor & sessionExecutor, uint64_t domainPartSize, - DomainPart::Crc defaultCrcType, const common::FileHeaderContext &fileHeaderContext); + Encoding defaultEncoding, const common::FileHeaderContext &fileHeaderContext); ~Domain() override; @@ -115,7 +115,7 @@ private: std::unique_ptr<Chunk> _currentChunk; SerialNum _lastSerial; - DomainPart::Crc _defaultCrcType; + Encoding _defaultEncoding; FastOS_ThreadPool & _threadPool; Executor & _commitExecutor; Executor & _sessionExecutor; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 0c79b922bb3..bc85e6b0d78 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -251,9 +251,9 @@ DomainPart::buildPacketMapping(bool allowTruncate) return currPos; } -DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Crc defaultCrc, +DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding defaultEncoding, const FileHeaderContext &fileHeaderContext, bool allowTruncate) : - _defaultCrc(defaultCrc), + _defaultEncoding(defaultEncoding), _lock(), _fileLock(), _range(s), @@ -535,12 +535,12 @@ DomainPart::write(FastOS_FileInterface &file, const Packet::Entry &entry) int32_t crc(0); uint32_t len(entry.serializedSize() + sizeof(crc)); nbostream os; - os << static_cast<uint8_t>(_defaultCrc); + os << _defaultEncoding.getRaw(); os << len; size_t start(os.size()); entry.serialize(os); size_t end(os.size()); - crc = Encoding::calcCrc(_defaultCrc, os.c_str()+start, end - start); + crc = Encoding::calcCrc(_defaultEncoding.getCrc(), os.c_str()+start, end - start); os << crc; size_t osSize = os.size(); assert(osSize == len + sizeof(len) + sizeof(uint8_t)); diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h index ef630904100..b62063d4466 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h @@ -20,9 +20,8 @@ private: DomainPart& operator=(const DomainPart &); public: - using Crc = Encoding::Crc; typedef std::shared_ptr<DomainPart> SP; - DomainPart(const vespalib::string &name, const vespalib::string &baseDir, SerialNum s, Crc defaultCrc, + DomainPart(const vespalib::string &name, const vespalib::string &baseDir, SerialNum s, Encoding defaultEncoding, const common::FileHeaderContext &FileHeaderContext, bool allowTruncate); ~DomainPart(); @@ -75,7 +74,7 @@ private: }; typedef std::vector<SkipInfo> SkipList; typedef std::map<SerialNum, Packet> PacketList; - const Crc _defaultCrc; + const Encoding _defaultEncoding; vespalib::Lock _lock; vespalib::Lock _fileLock; SerialNumRange _range; diff --git a/searchlib/src/vespa/searchlib/transactionlog/ichunk.h b/searchlib/src/vespa/searchlib/transactionlog/ichunk.h index 7e4787b79ab..6a5c7703d2d 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/ichunk.h +++ b/searchlib/src/vespa/searchlib/transactionlog/ichunk.h @@ -25,6 +25,7 @@ public: Crc getCrc() const { return Crc(_raw & 0x3); } Compression getCompression() const { return Compression((_raw >> 2) & 0xf); } static int32_t calcCrc(Crc version, const void * buf, size_t sz); + uint8_t getRaw() const { return _raw; } private: uint8_t _raw; }; diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index cf2b1510526..db5b0cd0607 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -78,17 +78,17 @@ TransLogServer::TransLogServer(const vespalib::string &name, int listenPort, con TransLogServer::TransLogServer(const vespalib::string &name, int listenPort, const vespalib::string &baseDir, const FileHeaderContext &fileHeaderContext, uint64_t domainPartSize) - : TransLogServer(name, listenPort, baseDir, fileHeaderContext, domainPartSize, 4, DomainPart::Crc::xxh64) + : TransLogServer(name, listenPort, baseDir, fileHeaderContext, domainPartSize, 4, Encoding::xxh64) {} TransLogServer::TransLogServer(const vespalib::string &name, int listenPort, const vespalib::string &baseDir, const FileHeaderContext &fileHeaderContext, uint64_t domainPartSize, - size_t maxThreads, DomainPart::Crc defaultCrcType) + size_t maxThreads, Encoding defaultCrcType) : FRT_Invokable(), _name(name), _baseDir(baseDir), _domainPartSize(domainPartSize), - _defaultCrcType(defaultCrcType), + _defaultEncoding(defaultCrcType), _commitExecutor(maxThreads, 128*1024), _sessionExecutor(maxThreads, 128*1024), _threadPool(0x20000), @@ -107,7 +107,7 @@ TransLogServer::TransLogServer(const vespalib::string &name, int listenPort, con if ( ! domainName.empty()) { try { auto domain = make_shared<Domain>(domainName, dir(), _threadPool, _commitExecutor, _sessionExecutor, - _domainPartSize, _defaultCrcType,_fileHeaderContext); + _domainPartSize, _defaultEncoding,_fileHeaderContext); _domains[domain->name()] = domain; } catch (const std::exception & e) { LOG(warning, "Failed creating %s domain on startup. Exception = %s", domainName.c_str(), e.what()); @@ -346,7 +346,7 @@ void TransLogServer::createDomain(FRT_RPCRequest *req) if ( !domain ) { try { domain = make_shared<Domain>(domainName, dir(), _threadPool, _commitExecutor, _sessionExecutor, - _domainPartSize, _defaultCrcType, _fileHeaderContext); + _domainPartSize, _defaultEncoding, _fileHeaderContext); { Guard domainGuard(_lock); _domains[domain->name()] = domain; diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h index d78d3d39887..225412cdaf6 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h @@ -23,7 +23,7 @@ public: TransLogServer(const vespalib::string &name, int listenPort, const vespalib::string &baseDir, const common::FileHeaderContext &fileHeaderContext, - uint64_t domainPartSize, size_t maxThreads, DomainPart::Crc defaultCrc); + uint64_t domainPartSize, size_t maxThreads, Encoding defaultCrc); TransLogServer(const vespalib::string &name, int listenPort, const vespalib::string &baseDir, const common::FileHeaderContext &fileHeaderContext, uint64_t domainPartSize); TransLogServer(const vespalib::string &name, int listenPort, const vespalib::string &baseDir, @@ -80,7 +80,7 @@ private: vespalib::string _name; vespalib::string _baseDir; const uint64_t _domainPartSize; - const DomainPart::Crc _defaultCrcType; + const Encoding _defaultEncoding; vespalib::ThreadStackExecutor _commitExecutor; vespalib::ThreadStackExecutor _sessionExecutor; FastOS_ThreadPool _threadPool; diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp index c6a4342632c..e21d6b1af62 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp @@ -23,13 +23,16 @@ TransLogServerApp::TransLogServerApp(const config::ConfigUri & tlsConfigUri, namespace { -DomainPart::Crc getCrc(searchlib::TranslogserverConfig::Crcmethod crcType) +Encoding +getEncoding(searchlib::TranslogserverConfig::Crcmethod crcType) { switch (crcType) { case searchlib::TranslogserverConfig::ccitt_crc32: - return Encoding::Crc::ccitt_crc32; + return Encoding(Encoding::Crc::ccitt_crc32, Encoding::Compression::none); case searchlib::TranslogserverConfig::xxh64: - return Encoding::Crc::xxh64; + return Encoding(Encoding::Crc::xxh64, Encoding::Compression::none); + case searchlib::TranslogserverConfig::xxh64lz4: + return Encoding(Encoding::Crc::xxh64, Encoding::Compression::lz4); } abort(); } @@ -40,7 +43,7 @@ void TransLogServerApp::start() { std::shared_ptr<searchlib::TranslogserverConfig> c = _tlsConfig.get(); _tls.reset(new TransLogServer(c->servername, c->listenport, c->basedir, _fileHeaderContext, - c->filesizemax, c->maxthreads, getCrc(c->crcmethod))); + c->filesizemax, c->maxthreads, getEncoding(c->crcmethod))); } TransLogServerApp::~TransLogServerApp() |