diff options
10 files changed, 85 insertions, 64 deletions
diff --git a/document/src/tests/serialization/compression_test.cpp b/document/src/tests/serialization/compression_test.cpp index b41dfad7b36..e5464100673 100644 --- a/document/src/tests/serialization/compression_test.cpp +++ b/document/src/tests/serialization/compression_test.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/document/util/compressor.h> +#include <vespa/vespalib/data/databuffer.h> LOG_SETUP("compression_test"); diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt index d8fe0d833ff..8d1275d0207 100644 --- a/document/src/vespa/document/util/CMakeLists.txt +++ b/document/src/vespa/document/util/CMakeLists.txt @@ -1,11 +1,12 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(document_util OBJECT SOURCES - stringutil.cpp bytebuffer.cpp - serializable.cpp compressor.cpp + lz4compressor.cpp printable.cpp + serializable.cpp + stringutil.cpp DEPENDS AFTER document_documentconfig diff --git a/document/src/vespa/document/util/compressor.cpp b/document/src/vespa/document/util/compressor.cpp index 0783bb28a60..d29f1db71b2 100644 --- a/document/src/vespa/document/util/compressor.cpp +++ b/document/src/vespa/document/util/compressor.cpp @@ -1,56 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "compressor.h" + +#include "lz4compressor.h" #include <vespa/vespalib/util/memory.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/data/databuffer.h> #include <stdexcept> -#include <lz4.h> -#include <lz4hc.h> using vespalib::alloc::Alloc; using vespalib::ConstBufferRef; using vespalib::DataBuffer; using vespalib::make_string; -namespace document -{ - -size_t LZ4Compressor::adjustProcessLen(uint16_t, size_t len) const { return LZ4_compressBound(len); } -size_t LZ4Compressor::adjustUnProcessLen(uint16_t, size_t len) const { return len; } - -bool -LZ4Compressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) -{ - const char * input(static_cast<const char *>(inputV)); - char * output(static_cast<char *>(outputV)); - int sz(-1); - int maxOutputLen = LZ4_compressBound(inputLen); - if (config.compressionLevel > 6) { - Alloc state = Alloc::alloc(LZ4_sizeofStateHC()); - sz = LZ4_compress_HC_extStateHC(state.get(), input, output, inputLen, maxOutputLen, config.compressionLevel); - } else { - Alloc state = Alloc::alloc(LZ4_sizeofState()); - sz = LZ4_compress_fast_extState(state.get(), input, output, inputLen, maxOutputLen, 1); - } - if (sz != 0) { - outputLenV = sz; - } - assert(sz != 0); - return (sz != 0); - -} - -bool -LZ4Compressor::unprocess(const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) -{ - const char * input(static_cast<const char *>(inputV)); - char * output(static_cast<char *>(outputV)); - int sz = LZ4_decompress_safe(input, output, inputLen, outputLenV); - if (sz > 0) { - outputLenV = sz; - } - assert(sz > 0); - return (sz > 0); -} +namespace document { CompressionConfig::Type compress(ICompressor & compressor, const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest) diff --git a/document/src/vespa/document/util/compressor.h b/document/src/vespa/document/util/compressor.h index 599d9424889..31e265415b7 100644 --- a/document/src/vespa/document/util/compressor.h +++ b/document/src/vespa/document/util/compressor.h @@ -1,12 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/util/compressionconfig.h> -#include <vespa/vespalib/data/databuffer.h> +#include "compressionconfig.h" #include <vespa/vespalib/util/buffer.h> -namespace document -{ +namespace vespalib { class DataBuffer; } + +namespace document { class ICompressor { @@ -18,15 +18,6 @@ public: virtual size_t adjustUnProcessLen(uint16_t options, size_t len) const = 0; }; -class LZ4Compressor : public ICompressor -{ -public: - bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) override; - bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) override; - size_t adjustProcessLen(uint16_t options, size_t len) const override; - size_t adjustUnProcessLen(uint16_t options, size_t len) const override; -}; - /** * Will try to compress a buffer according to the config. If the criteria can not * be met it will return NONE and dest will get the input buffer. @@ -52,4 +43,3 @@ CompressionConfig::Type compress(const CompressionConfig & compression, const ve void decompress(const CompressionConfig::Type & compression, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap); } - diff --git a/document/src/vespa/document/util/lz4compressor.cpp b/document/src/vespa/document/util/lz4compressor.cpp new file mode 100644 index 00000000000..cb2d41269b0 --- /dev/null +++ b/document/src/vespa/document/util/lz4compressor.cpp @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "lz4compressor.h" +#include <vespa/vespalib/util/alloc.h> +#include <lz4.h> +#include <lz4hc.h> +#include <cassert> + +using vespalib::alloc::Alloc; + + +namespace document { + +size_t LZ4Compressor::adjustProcessLen(uint16_t, size_t len) const { return LZ4_compressBound(len); } +size_t LZ4Compressor::adjustUnProcessLen(uint16_t, size_t len) const { return len; } + +bool +LZ4Compressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) +{ + const char * input(static_cast<const char *>(inputV)); + char * output(static_cast<char *>(outputV)); + int sz(-1); + int maxOutputLen = LZ4_compressBound(inputLen); + if (config.compressionLevel > 6) { + Alloc state = Alloc::alloc(LZ4_sizeofStateHC()); + sz = LZ4_compress_HC_extStateHC(state.get(), input, output, inputLen, maxOutputLen, config.compressionLevel); + } else { + Alloc state = Alloc::alloc(LZ4_sizeofState()); + sz = LZ4_compress_fast_extState(state.get(), input, output, inputLen, maxOutputLen, 1); + } + if (sz != 0) { + outputLenV = sz; + } + assert(sz != 0); + return (sz != 0); + +} + +bool +LZ4Compressor::unprocess(const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) +{ + const char * input(static_cast<const char *>(inputV)); + char * output(static_cast<char *>(outputV)); + int sz = LZ4_decompress_safe(input, output, inputLen, outputLenV); + if (sz > 0) { + outputLenV = sz; + } + assert(sz > 0); + return (sz > 0); +} + +} diff --git a/document/src/vespa/document/util/lz4compressor.h b/document/src/vespa/document/util/lz4compressor.h new file mode 100644 index 00000000000..0c3eee82402 --- /dev/null +++ b/document/src/vespa/document/util/lz4compressor.h @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "compressor.h" + +namespace document { + +class LZ4Compressor : public ICompressor +{ +public: + bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) override; + bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) override; + size_t adjustProcessLen(uint16_t options, size_t len) const override; + size_t adjustUnProcessLen(uint16_t options, size_t len) const override; +}; + +} + diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp index 99b39f68006..8fffc4bb831 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp @@ -4,9 +4,9 @@ #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/common/packets.h> #include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/log/log.h> - LOG_SETUP(".proton.summaryengine.docsum_by_slime"); namespace proton { diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 531305789b3..50a02995a40 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -7,6 +7,7 @@ #include <vespa/document/util/compressor.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.common.fs4packets"); diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index cddf8c96b2e..0a77da01785 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "chunkformats.h" -#include <vespa/document/util/compressor.h> +#include <vespa/document/util/lz4compressor.h> #include <vespa/vespalib/util/stringfmt.h> namespace search { diff --git a/vespalog/src/vespa/log/bufferedlogger.cpp b/vespalog/src/vespa/log/bufferedlogger.cpp index db2021324c3..fe6a03ebf69 100644 --- a/vespalog/src/vespa/log/bufferedlogger.cpp +++ b/vespalog/src/vespa/log/bufferedlogger.cpp @@ -10,11 +10,9 @@ #include <boost/multi_index/sequenced_index.hpp> #include <iomanip> -#include <iostream> #include <sstream> #include <vector> #include <cstdarg> -#include <ctime> namespace ns_log { @@ -381,5 +379,4 @@ BufferedLogger::setTimer(std::unique_ptr<Timer> timer) _backing->_timer = std::move(timer); } - } // ns_log |