Combine crc and compression.

10 files changed, 32 insertions, 29 deletions

diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp
index b389c1a8da5..9589d43c762 100644
--- a/searchlib/src/tests/transactionlog/translogclient_test.cpp
+++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp
@@ -44,7 +44,7 @@ private:
     bool partialUpdateTest();
     bool testVisitOverGeneratedDomain();
     bool testRemove();
-    void createAndFillDomain(const vespalib::string & name, DomainPart::Crc crcMethod, size_t preExistingDomains);
+    void createAndFillDomain(const vespalib::string & name, Encoding encoding, size_t preExistingDomains);
     void verifyDomain(const vespalib::string & name);
     void testCrcVersions();
     bool testVisitOverPreExistingDomain();
@@ -491,10 +491,10 @@ bool Test::testVisitOverGeneratedDomain()
     return true;
 }
 
-void Test::createAndFillDomain(const vespalib::string & name, DomainPart::Crc crcMethod, size_t preExistingDomains)
+void Test::createAndFillDomain(const vespalib::string & name, Encoding encoding, size_t preExistingDomains)
 {
     DummyFileHeaderContext fileHeaderContext;
-    TransLogServer tlss("test13", 18377, ".", fileHeaderContext, 0x10000, 4, crcMethod);
+    TransLogServer tlss("test13", 18377, ".", fileHeaderContext, 0x10000, 4, encoding);
     TransLogClient tls("tcp/localhost:18377");
 
     createDomainTest(tls, name, preExistingDomains);
diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def
index 74efe3fe68e..9d5e080e138 100644
--- a/searchlib/src/vespa/searchlib/config/translogserver.def
+++ b/searchlib/src/vespa/searchlib/config/translogserver.def
@@ -21,4 +21,4 @@ usefsync bool default=false restart
 maxthreads int default=4 restart
 
 ##Default crc method used
-crcmethod enum {ccitt_crc32, xxh64} default=xxh64
+crcmethod enum {ccitt_crc32, xxh64, xxh64lz4} default=xxh64
diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp
index ea8961dc564..c3b2738eac1 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp
+++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp
@@ -26,10 +26,10 @@ namespace search::transactionlog {
 
 Domain::Domain(const string &domainName, const string & baseDir, FastOS_ThreadPool & threadPool,
                Executor & commitExecutor, Executor & sessionExecutor, uint64_t domainPartSize,
-               DomainPart::Crc defaultCrcType, const FileHeaderContext &fileHeaderContext) :
+               Encoding defaultEncoding, const FileHeaderContext &fileHeaderContext) :
     _currentChunk(std::make_unique<Chunk>()),
     _lastSerial(0),
-    _defaultCrcType(defaultCrcType),
+    _defaultEncoding(defaultEncoding),
     _threadPool(threadPool),
     _commitExecutor(commitExecutor),
     _sessionExecutor(sessionExecutor),
@@ -67,7 +67,7 @@ Domain::Domain(const string &domainName, const string & baseDir, FastOS_ThreadPo
     }
     _sessionExecutor.sync();
     if (_parts.empty() || _parts.crbegin()->second->isClosed()) {
-        _parts[lastPart].reset(new DomainPart(_name, dir(), lastPart, _defaultCrcType, _fileHeaderContext, false));
+        _parts[lastPart].reset(new DomainPart(_name, dir(), lastPart, _defaultEncoding, _fileHeaderContext, false));
     }
     _lastSerial = end();
     _self = _threadPool.NewThread(this);
@@ -84,7 +84,7 @@ Domain::Run(FastOS_ThreadInterface *thisThread, void *) {
     }
 }
 void Domain::addPart(int64_t partId, bool isLastPart) {
-    DomainPart::SP dp(new DomainPart(_name, dir(), partId, _defaultCrcType, _fileHeaderContext, isLastPart));
+    DomainPart::SP dp(new DomainPart(_name, dir(), partId, _defaultEncoding, _fileHeaderContext, isLastPart));
     if (dp->size() == 0) {
         // Only last domain part is allowed to be truncated down to
         // empty size.
@@ -381,7 +381,7 @@ Domain::commitChunk(std::unique_ptr<Chunk> chunk, const vespalib::MonitorGuard &
         triggerSyncNow();
         waitPendingSync(_syncMonitor, _pendingSync);
         dp->close();
-        dp.reset(new DomainPart(_name, dir(), entry.serial(), _defaultCrcType, _fileHeaderContext, false));
+        dp.reset(new DomainPart(_name, dir(), entry.serial(), _defaultEncoding, _fileHeaderContext, false));
         {
             LockGuard guard(_lock);
             _parts[entry.serial()] = dp;
diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h
index fb18bb955d7..c07459b0da9 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/domain.h
+++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h
@@ -42,7 +42,7 @@ public:
     using Executor = vespalib::ThreadExecutor;
     Domain(const vespalib::string &name, const vespalib::string &baseDir, FastOS_ThreadPool & threadPool,
            Executor & commitExecutor, Executor & sessionExecutor, uint64_t domainPartSize,
-           DomainPart::Crc defaultCrcType, const common::FileHeaderContext &fileHeaderContext);
+           Encoding defaultEncoding, const common::FileHeaderContext &fileHeaderContext);
 
     ~Domain() override;
 
@@ -115,7 +115,7 @@ private:
 
     std::unique_ptr<Chunk> _currentChunk;
     SerialNum           _lastSerial;
-    DomainPart::Crc     _defaultCrcType;
+    Encoding            _defaultEncoding;
     FastOS_ThreadPool & _threadPool;
     Executor          & _commitExecutor;
     Executor          & _sessionExecutor;
diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp
index 0c79b922bb3..bc85e6b0d78 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp
+++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp
@@ -251,9 +251,9 @@ DomainPart::buildPacketMapping(bool allowTruncate)
     return currPos;
 }
 
-DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Crc defaultCrc,
+DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding defaultEncoding,
                        const FileHeaderContext &fileHeaderContext, bool allowTruncate) :
-    _defaultCrc(defaultCrc),
+    _defaultEncoding(defaultEncoding),
     _lock(),
     _fileLock(),
     _range(s),
@@ -535,12 +535,12 @@ DomainPart::write(FastOS_FileInterface &file, const Packet::Entry &entry)
     int32_t crc(0);
     uint32_t len(entry.serializedSize() + sizeof(crc));
     nbostream os;
-    os << static_cast<uint8_t>(_defaultCrc);
+    os << _defaultEncoding.getRaw();
     os << len;
     size_t start(os.size());
     entry.serialize(os);
     size_t end(os.size());
-    crc = Encoding::calcCrc(_defaultCrc, os.c_str()+start, end - start);
+    crc = Encoding::calcCrc(_defaultEncoding.getCrc(), os.c_str()+start, end - start);
     os << crc;
     size_t osSize = os.size();
     assert(osSize == len + sizeof(len) + sizeof(uint8_t));
diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h
index ef630904100..b62063d4466 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h
+++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h
@@ -20,9 +20,8 @@ private:
     DomainPart& operator=(const DomainPart &);
 
 public:
-    using Crc = Encoding::Crc;
     typedef std::shared_ptr<DomainPart> SP;
-    DomainPart(const vespalib::string &name, const vespalib::string &baseDir, SerialNum s, Crc defaultCrc,
+    DomainPart(const vespalib::string &name, const vespalib::string &baseDir, SerialNum s, Encoding defaultEncoding,
                const common::FileHeaderContext &FileHeaderContext, bool allowTruncate);
 
     ~DomainPart();
@@ -75,7 +74,7 @@ private:
     };
     typedef std::vector<SkipInfo> SkipList;
     typedef std::map<SerialNum, Packet> PacketList;
-    const Crc             _defaultCrc;
+    const Encoding        _defaultEncoding;
     vespalib::Lock        _lock;
     vespalib::Lock        _fileLock;
     SerialNumRange        _range;
diff --git a/searchlib/src/vespa/searchlib/transactionlog/ichunk.h b/searchlib/src/vespa/searchlib/transactionlog/ichunk.h
index 7e4787b79ab..6a5c7703d2d 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/ichunk.h
+++ b/searchlib/src/vespa/searchlib/transactionlog/ichunk.h
@@ -25,6 +25,7 @@ public:
     Crc getCrc() const { return Crc(_raw & 0x3); }
     Compression getCompression() const { return Compression((_raw >> 2) & 0xf); }
     static int32_t calcCrc(Crc version, const void * buf, size_t sz);
+    uint8_t getRaw() const { return _raw; }
 private:
     uint8_t _raw;
 };
diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp
index cf2b1510526..db5b0cd0607 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp
+++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp
@@ -78,17 +78,17 @@ TransLogServer::TransLogServer(const vespalib::string &name, int listenPort, con
 
 TransLogServer::TransLogServer(const vespalib::string &name, int listenPort, const vespalib::string &baseDir,
                                const FileHeaderContext &fileHeaderContext, uint64_t domainPartSize)
-    : TransLogServer(name, listenPort, baseDir, fileHeaderContext, domainPartSize, 4, DomainPart::Crc::xxh64)
+    : TransLogServer(name, listenPort, baseDir, fileHeaderContext, domainPartSize, 4, Encoding::xxh64)
 {}
 
 TransLogServer::TransLogServer(const vespalib::string &name, int listenPort, const vespalib::string &baseDir,
                                const FileHeaderContext &fileHeaderContext, uint64_t domainPartSize,
-                               size_t maxThreads, DomainPart::Crc defaultCrcType)
+                               size_t maxThreads, Encoding defaultCrcType)
     : FRT_Invokable(),
       _name(name),
       _baseDir(baseDir),
       _domainPartSize(domainPartSize),
-      _defaultCrcType(defaultCrcType),
+      _defaultEncoding(defaultCrcType),
       _commitExecutor(maxThreads, 128*1024),
       _sessionExecutor(maxThreads, 128*1024),
       _threadPool(0x20000),
@@ -107,7 +107,7 @@ TransLogServer::TransLogServer(const vespalib::string &name, int listenPort, con
                 if ( ! domainName.empty()) {
                     try {
                         auto domain = make_shared<Domain>(domainName, dir(), _threadPool, _commitExecutor, _sessionExecutor,
-                                                          _domainPartSize, _defaultCrcType,_fileHeaderContext);
+                                                          _domainPartSize, _defaultEncoding,_fileHeaderContext);
                         _domains[domain->name()] = domain;
                     } catch (const std::exception & e) {
                         LOG(warning, "Failed creating %s domain on startup. Exception = %s", domainName.c_str(), e.what());
@@ -346,7 +346,7 @@ void TransLogServer::createDomain(FRT_RPCRequest *req)
     if ( !domain ) {
         try {
             domain = make_shared<Domain>(domainName, dir(), _threadPool, _commitExecutor, _sessionExecutor,
-                                         _domainPartSize, _defaultCrcType, _fileHeaderContext);
+                                         _domainPartSize, _defaultEncoding, _fileHeaderContext);
             {
                 Guard domainGuard(_lock);
                 _domains[domain->name()] = domain;
diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h
index d78d3d39887..225412cdaf6 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h
+++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h
@@ -23,7 +23,7 @@ public:
 
     TransLogServer(const vespalib::string &name, int listenPort, const vespalib::string &baseDir,
                    const common::FileHeaderContext &fileHeaderContext,
-                   uint64_t domainPartSize, size_t maxThreads, DomainPart::Crc defaultCrc);
+                   uint64_t domainPartSize, size_t maxThreads, Encoding defaultCrc);
     TransLogServer(const vespalib::string &name, int listenPort, const vespalib::string &baseDir,
                    const common::FileHeaderContext &fileHeaderContext, uint64_t domainPartSize);
     TransLogServer(const vespalib::string &name, int listenPort, const vespalib::string &baseDir,
@@ -80,7 +80,7 @@ private:
     vespalib::string                    _name;
     vespalib::string                    _baseDir;
     const uint64_t                      _domainPartSize;
-    const DomainPart::Crc               _defaultCrcType;
+    const Encoding                      _defaultEncoding;
     vespalib::ThreadStackExecutor       _commitExecutor;
     vespalib::ThreadStackExecutor       _sessionExecutor;
     FastOS_ThreadPool                   _threadPool;
diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp
index c6a4342632c..e21d6b1af62 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp
+++ b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp
@@ -23,13 +23,16 @@ TransLogServerApp::TransLogServerApp(const config::ConfigUri & tlsConfigUri,
 
 namespace {
 
-DomainPart::Crc getCrc(searchlib::TranslogserverConfig::Crcmethod crcType)
+Encoding
+getEncoding(searchlib::TranslogserverConfig::Crcmethod crcType)
 {
     switch (crcType) {
         case searchlib::TranslogserverConfig::ccitt_crc32:
-            return Encoding::Crc::ccitt_crc32;
+            return Encoding(Encoding::Crc::ccitt_crc32, Encoding::Compression::none);
         case searchlib::TranslogserverConfig::xxh64:
-            return Encoding::Crc::xxh64;
+            return Encoding(Encoding::Crc::xxh64, Encoding::Compression::none);
+        case searchlib::TranslogserverConfig::xxh64lz4:
+            return Encoding(Encoding::Crc::xxh64, Encoding::Compression::lz4);
     }
     abort();
 }
@@ -40,7 +43,7 @@ void TransLogServerApp::start()
 {
     std::shared_ptr<searchlib::TranslogserverConfig> c = _tlsConfig.get();
     _tls.reset(new TransLogServer(c->servername, c->listenport, c->basedir, _fileHeaderContext,
-                                  c->filesizemax, c->maxthreads, getCrc(c->crcmethod)));
+                                  c->filesizemax, c->maxthreads, getEncoding(c->crcmethod)));
 }
 
 TransLogServerApp::~TransLogServerApp()