Merge pull request #2720 from yahoo/balder/add-zstd-compressor

Balder/add zstd compressor

5 files changed, 15 insertions, 8 deletions

diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
index 80b78d420ec..dab4dcf1dff 100644
--- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
+++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
@@ -381,7 +381,7 @@ verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t org
 
     DataBuffer uncompressed;
     ConstBufferRef blob(ret[2]._data._buf, ret[2]._data._len);
-    document::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, blob, uncompressed, false);
+    compression::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, blob, uncompressed, false);
     EXPECT_EQUAL(orgSize, uncompressed.getDataLen());
 
     vespalib::Slime summaries;
@@ -402,7 +402,7 @@ verifyRPC(size_t count,
 
     CompressionConfig config(requestCompression, 9, 100);
     DataBuffer compressed(const_cast<char *>(buf.get().data), buf.get().size);
-    CompressionConfig::Type type = document::compress(config, ConstBufferRef(buf.get().data, buf.get().size), compressed, true);
+    CompressionConfig::Type type = compression::compress(config, ConstBufferRef(buf.get().data, buf.get().size), compressed, true);
     EXPECT_EQUAL(type, requestCompression);
 
     FRT_RPCRequest * request = new FRT_RPCRequest();
diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def
index c851d124970..8c4e7b4571a 100644
--- a/searchcore/src/vespa/searchcore/config/proton.def
+++ b/searchcore/src/vespa/searchcore/config/proton.def
@@ -210,21 +210,21 @@ summary.cache.allowvisitcaching bool default=false restart
 summary.cache.initialentries long default=0 restart
 
 ## Control compression type of the summary while in the cache.
-summary.cache.compression.type enum {NONE, LZ4} default=LZ4 restart
+summary.cache.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 restart
 
 ## Control compression level of the summary while in cache.
 summary.cache.compression.level int default=9 restart
 
 ## Control compression type of the summary while in memory during compaction
 ## NB So far only stragey=LOG honours it.
-summary.log.compact.compression.type enum {NONE, LZ4} default=LZ4 restart
+summary.log.compact.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 restart
 
 ## Control compression level of the summary while in memory during compaction
 summary.log.compact.compression.level int default=9 restart
 
 ## Control compression type of the summary
 ## NB So far only stragey=LOG honours it.
-summary.log.chunk.compression.type enum {NONE, LZ4} default=LZ4 restart
+summary.log.chunk.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 restart
 
 ## Control compression level of the summary
 summary.log.chunk.compression.level int default=9 restart
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
index 0253e943883..e5a6d6aed22 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
@@ -117,6 +117,8 @@ deriveCompression(const T & config) {
     document::CompressionConfig compression;
     if (config.type == T::LZ4) {
         compression.type = document::CompressionConfig::LZ4;
+    } else if (config.type == T::ZSTD) {
+        compression.type = document::CompressionConfig::ZSTD;
     }
     compression.compressionLevel = config.level;
     return compression;
@@ -160,6 +162,8 @@ SummaryManager::SummaryManager(vespalib::ThreadExecutor & executor,
                                          summary.compact2buckets ? bucketizer : search::IBucketizer::SP()));
 }
 
+SummaryManager::~SummaryManager() {}
+
 void
 SummaryManager::putDocument(uint64_t syncToken, const Document & doc, search::DocumentIdT lid)
 {
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h
index b55345ae470..a1bcd34fd0f 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h
@@ -82,6 +82,7 @@ public:
                    const search::common::FileHeaderContext &fileHeaderContext,
                    search::transactionlog::SyncProxy &tlSyncer,
                    const std::shared_ptr<search::IBucketizer> & bucketizer);
+    ~SummaryManager();
 
     void putDocument(uint64_t syncToken, const document::Document & doc,
                      search::DocumentIdT lid);
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp
index 99b39f68006..465c6fa9d03 100644
--- a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp
@@ -4,9 +4,9 @@
 #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h>
 #include <vespa/searchlib/common/packets.h>
 #include <vespa/fnet/frt/rpcrequest.h>
+#include <vespa/vespalib/data/databuffer.h>
 
 #include <vespa/log/log.h>
-
 LOG_SETUP(".proton.summaryengine.docsum_by_slime");
 
 namespace proton {
@@ -90,12 +90,14 @@ DocsumByRPC::DocsumByRPC(DocsumBySlime & slimeDocsumServer) :
 void
 DocsumByRPC::getDocsums(FRT_RPCRequest & req)
 {
+    using document::compression::decompress;
+    using document::compression::compress;
     FRT_Values &arg = *req.GetParams();
     uint8_t encoding = arg[0]._intval8;
     uint32_t uncompressedSize = arg[1]._intval32;
     DataBuffer uncompressed(arg[2]._data._buf, arg[2]._data._len);
     ConstBufferRef blob(arg[2]._data._buf, arg[2]._data._len);
-    document::decompress(CompressionConfig::toType(encoding), uncompressedSize, blob, uncompressed, true);
+    decompress(CompressionConfig::toType(encoding), uncompressedSize, blob, uncompressed, true);
     assert(uncompressedSize == uncompressed.getDataLen());
     vespalib::Slime summariesToGet;
     BinaryFormat::decode(Memory(uncompressed.getData(), uncompressed.getDataLen()), summariesToGet);
@@ -108,7 +110,7 @@ DocsumByRPC::getDocsums(FRT_RPCRequest & req)
     BinaryFormat::encode(*summaries, output);
     ConstBufferRef buf(rbuf.GetDrainPos(), rbuf.GetUsedLen());
     DataBuffer compressed(rbuf.GetWritableDrainPos(0), rbuf.GetUsedLen());
-    CompressionConfig::Type type = document::compress(getCompressionConfig(), buf, compressed, true);
+    CompressionConfig::Type type = compress(getCompressionConfig(), buf, compressed, true);
 
     FRT_Values &ret = *req.GetReturn();
     ret.AddInt8(type);