diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-06-13 18:52:46 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-06-13 18:52:46 +0200 |
commit | 0155137e7d56730665d406c37aa75649e5a3418a (patch) | |
tree | 05100870215c7852cc27b6970b60c9ff5f77e763 /searchcore | |
parent | 4e5d87a9baa4176a1089f7afd1f433a8da275725 (diff) | |
parent | 6a64952133fc05136a0e445851b49bd0fc71c00e (diff) |
Merge with master
Diffstat (limited to 'searchcore')
5 files changed, 15 insertions, 8 deletions
diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index 69af759db40..3262d571c56 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -381,7 +381,7 @@ verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t org DataBuffer uncompressed; ConstBufferRef blob(ret[2]._data._buf, ret[2]._data._len); - document::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, blob, uncompressed, false); + compression::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, blob, uncompressed, false); EXPECT_EQUAL(orgSize, uncompressed.getDataLen()); vespalib::Slime summaries; @@ -402,7 +402,7 @@ verifyRPC(size_t count, CompressionConfig config(requestCompression, 9, 100); DataBuffer compressed(const_cast<char *>(buf.get().data), buf.get().size); - CompressionConfig::Type type = document::compress(config, ConstBufferRef(buf.get().data, buf.get().size), compressed, true); + CompressionConfig::Type type = compression::compress(config, ConstBufferRef(buf.get().data, buf.get().size), compressed, true); EXPECT_EQUAL(type, requestCompression); FRT_RPCRequest * request = new FRT_RPCRequest(); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 58ff711aa60..deb091b42da 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -210,21 +210,21 @@ summary.cache.allowvisitcaching bool default=false restart summary.cache.initialentries long default=0 restart ## Control compression type of the summary while in the cache. -summary.cache.compression.type enum {NONE, LZ4} default=LZ4 restart +summary.cache.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 restart ## Control compression level of the summary while in cache. summary.cache.compression.level int default=9 restart ## Control compression type of the summary while in memory during compaction ## NB So far only stragey=LOG honours it. -summary.log.compact.compression.type enum {NONE, LZ4} default=LZ4 restart +summary.log.compact.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 restart ## Control compression level of the summary while in memory during compaction summary.log.compact.compression.level int default=9 restart ## Control compression type of the summary ## NB So far only stragey=LOG honours it. -summary.log.chunk.compression.type enum {NONE, LZ4} default=LZ4 restart +summary.log.chunk.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 restart ## Control compression level of the summary summary.log.chunk.compression.level int default=9 restart diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 343c051cb5c..014e861f4fa 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -117,6 +117,8 @@ deriveCompression(const T & config) { document::CompressionConfig compression; if (config.type == T::LZ4) { compression.type = document::CompressionConfig::LZ4; + } else if (config.type == T::ZSTD) { + compression.type = document::CompressionConfig::ZSTD; } compression.compressionLevel = config.level; return compression; @@ -160,6 +162,8 @@ SummaryManager::SummaryManager(vespalib::ThreadExecutor & executor, summary.compact2buckets ? bucketizer : search::IBucketizer::SP())); } +SummaryManager::~SummaryManager() {} + void SummaryManager::putDocument(uint64_t syncToken, const Document & doc, search::DocumentIdT lid) { diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h index 3c2b6a204e7..bb5f4354845 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h @@ -82,6 +82,7 @@ public: const search::common::FileHeaderContext &fileHeaderContext, search::transactionlog::SyncProxy &tlSyncer, const std::shared_ptr<search::IBucketizer> & bucketizer); + ~SummaryManager(); void putDocument(uint64_t syncToken, const document::Document & doc, search::DocumentIdT lid); diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp index cc10a3aab43..8dcd1381b20 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp @@ -4,9 +4,9 @@ #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/common/packets.h> #include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/log/log.h> - LOG_SETUP(".proton.summaryengine.docsum_by_slime"); namespace proton { @@ -90,12 +90,14 @@ DocsumByRPC::DocsumByRPC(DocsumBySlime & slimeDocsumServer) : void DocsumByRPC::getDocsums(FRT_RPCRequest & req) { + using document::compression::decompress; + using document::compression::compress; FRT_Values &arg = *req.GetParams(); uint8_t encoding = arg[0]._intval8; uint32_t uncompressedSize = arg[1]._intval32; DataBuffer uncompressed(arg[2]._data._buf, arg[2]._data._len); ConstBufferRef blob(arg[2]._data._buf, arg[2]._data._len); - document::decompress(CompressionConfig::toType(encoding), uncompressedSize, blob, uncompressed, true); + decompress(CompressionConfig::toType(encoding), uncompressedSize, blob, uncompressed, true); assert(uncompressedSize == uncompressed.getDataLen()); vespalib::Slime summariesToGet; BinaryFormat::decode(Memory(uncompressed.getData(), uncompressed.getDataLen()), summariesToGet); @@ -108,7 +110,7 @@ DocsumByRPC::getDocsums(FRT_RPCRequest & req) BinaryFormat::encode(*summaries, output); ConstBufferRef buf(rbuf.GetDrainPos(), rbuf.GetUsedLen()); DataBuffer compressed(rbuf.GetWritableDrainPos(0), rbuf.GetUsedLen()); - CompressionConfig::Type type = document::compress(getCompressionConfig(), buf, compressed, true); + CompressionConfig::Type type = compress(getCompressionConfig(), buf, compressed, true); FRT_Values &ret = *req.GetReturn(); ret.AddInt8(type); |