diff options
Diffstat (limited to 'document')
23 files changed, 22 insertions, 499 deletions
diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index e9694390b4b..ca2ee029c87 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -8,10 +8,6 @@ vespa_define_module( config_cloudconfig vespaeval - EXTERNAL_DEPENDS - lz4 - zstd - LIBS src/vespa/document src/vespa/document/annotation diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index 44cff2c80de..c3f88ac3696 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -16,6 +16,7 @@ #include <fcntl.h> using vespalib::nbostream; +using vespalib::compression::CompressionConfig; using namespace document::config_builder; diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp index 508e6f237cf..1e8fd9ec470 100644 --- a/document/src/tests/repo/documenttyperepo_test.cpp +++ b/document/src/tests/repo/documenttyperepo_test.cpp @@ -10,7 +10,6 @@ #include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> -#include <stdlib.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/testapp.h> @@ -25,6 +24,7 @@ using std::vector; using vespalib::Identifiable; using vespalib::IllegalArgumentException; using vespalib::string; +using vespalib::compression::CompressionConfig; using namespace document::config_builder; using namespace document; diff --git a/document/src/tests/serialization/.gitignore b/document/src/tests/serialization/.gitignore index 9f5bc440533..7e94d2757ae 100644 --- a/document/src/tests/serialization/.gitignore +++ b/document/src/tests/serialization/.gitignore @@ -2,5 +2,4 @@ .depend Makefile document_annotationserializer_test_app -document_compression_test_app document_vespadocumentserializer_test_app diff --git a/document/src/tests/serialization/CMakeLists.txt b/document/src/tests/serialization/CMakeLists.txt index 5b23bffad26..e1ce43b12d4 100644 --- a/document/src/tests/serialization/CMakeLists.txt +++ b/document/src/tests/serialization/CMakeLists.txt @@ -17,12 +17,3 @@ vespa_add_executable(document_annotationserializer_test_app TEST document_documentconfig ) vespa_add_test(NAME document_annotationserializer_test_app COMMAND document_annotationserializer_test_app) -vespa_add_executable(document_compression_test_app TEST - SOURCES - compression_test.cpp - DEPENDS - document - AFTER - document_documentconfig -) -vespa_add_test(NAME document_compression_test_app COMMAND document_compression_test_app) diff --git a/document/src/tests/serialization/compression_test.cpp b/document/src/tests/serialization/compression_test.cpp deleted file mode 100644 index 6574b4ac34c..00000000000 --- a/document/src/tests/serialization/compression_test.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/document/util/compressor.h> -#include <vespa/vespalib/data/databuffer.h> - -#include <vespa/log/log.h> -LOG_SETUP("compression_test"); - -using namespace document; -using namespace document::compression; -using namespace vespalib; - -static vespalib::string _G_compressableText("AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" - "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" - "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" - "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" - "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" - "XYZABCDEFGHIJGJMNOPQRSTUVW" - "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" - "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" - "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" - "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" - "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" - "XYZABCDEFGHIJGJMNOPQRSTUVW"); - -TEST("requireThatLZ4CompressFine") { - CompressionConfig cfg(CompressionConfig::Type::LZ4); - ConstBufferRef ref(_G_compressableText.c_str(), _G_compressableText.size()); - DataBuffer compressed; - EXPECT_EQUAL(CompressionConfig::Type::LZ4, compress(cfg, ref, compressed, false)); - EXPECT_EQUAL(66u, compressed.getDataLen()); -} - -TEST("requireThatZStdCompressFine") { - CompressionConfig cfg(CompressionConfig::Type::ZSTD); - ConstBufferRef ref(_G_compressableText.c_str(), _G_compressableText.size()); - DataBuffer compressed; - EXPECT_EQUAL(CompressionConfig::Type::ZSTD, compress(cfg, ref, compressed, false)); - EXPECT_EQUAL(64u, compressed.getDataLen()); -} - -TEST_MAIN() { - TEST_RUN_ALL(); -} diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index d09012a4e4b..9da20e5a84c 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -55,6 +55,7 @@ using vespalib::tensor::Tensor; using vespalib::tensor::TensorBuilder; using vespalib::tensor::TensorCells; using vespalib::tensor::TensorDimensions; +using vespalib::compression::CompressionConfig; using namespace document; using std::string; using std::vector; diff --git a/document/src/vespa/document/datatype/structdatatype.h b/document/src/vespa/document/datatype/structdatatype.h index f6c77f18f47..0f1c58316c9 100644 --- a/document/src/vespa/document/datatype/structdatatype.h +++ b/document/src/vespa/document/datatype/structdatatype.h @@ -11,15 +11,16 @@ #include <vespa/document/datatype/structureddatatype.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <memory> namespace document { class StructDataType final : public StructuredDataType { public: - typedef std::unique_ptr<StructDataType> UP; - typedef std::shared_ptr<StructDataType> SP; + using UP = std::unique_ptr<StructDataType>; + using SP = std::shared_ptr<StructDataType>; + using CompressionConfig = vespalib::compression::CompressionConfig; StructDataType(); StructDataType(const vespalib::stringref &name); diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index 76216d75f1b..5dfd8eff891 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -2,7 +2,7 @@ #include "serializablearray.h" #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/util/bytebuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/data/databuffer.h> #include <algorithm> @@ -181,7 +181,7 @@ SerializableArray::clear(int id) void SerializableArray::deCompress() // throw (DeserializeException) { - using document::compression::decompress; + using vespalib::compression::decompress; // will only do this once LOG_ASSERT(_compSerData); @@ -239,7 +239,7 @@ void SerializableArray::assign(EntryMap & entries, } } -CompressionInfo +vespalib::compression::CompressionInfo SerializableArray::getCompressionInfo() const { return CompressionInfo(_uncompressedLength, _compSerData->getRemaining()); } diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h index 2d12c1191a6..2f7d65938aa 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.h +++ b/document/src/vespa/document/fieldvalue/serializablearray.h @@ -16,7 +16,7 @@ #pragma once -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/objects/cloneable.h> #include <vespa/vespalib/util/buffer.h> #include <vespa/vespalib/util/memory.h> @@ -107,6 +107,8 @@ public: using CP = vespalib::CloneablePtr<SerializableArray>; using UP = std::unique_ptr<SerializableArray>; using ByteBufferUP = std::unique_ptr<ByteBuffer>; + using CompressionConfig = vespalib::compression::CompressionConfig; + using CompressionInfo = vespalib::compression::CompressionInfo; SerializableArray(); virtual ~SerializableArray(); diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index c0ae342fd34..0a05ae60600 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -21,6 +21,7 @@ using std::vector; using vespalib::nbostream; using vespalib::nbostream_longlivedbuf; using vespalib::make_string; +using vespalib::compression::CompressionConfig; using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index 6ab6f71ce56..bb6956cf012 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -52,7 +52,9 @@ private: mutable bool _hasChanged; public: - typedef std::unique_ptr<StructFieldValue> UP; + using UP = std::unique_ptr<StructFieldValue>; + using CompressionConfig = vespalib::compression::CompressionConfig; + StructFieldValue(const DataType &type); ~StructFieldValue(); void swap(StructFieldValue & rhs); diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index 870e88e5036..6bfae246c10 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -30,6 +30,7 @@ using vespalib::hash_map; using vespalib::make_string; using vespalib::string; using vespalib::stringref; +using vespalib::compression::CompressionConfig; namespace document { diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index bee1c258e4a..2b45e8a298c 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -38,6 +38,7 @@ using vespalib::asciistream; using vespalib::nbostream; using vespalib::Memory; using vespalib::stringref; +using vespalib::compression::CompressionConfig; namespace document { diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 2534690b014..6c9e95a9dd6 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -27,7 +27,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> using std::make_pair; using std::pair; @@ -36,6 +36,7 @@ using vespalib::nbostream; using vespalib::stringref; using vespalib::string; using vespalib::slime::BinaryFormat; +using vespalib::compression::CompressionConfig; namespace document { @@ -275,7 +276,7 @@ vespalib::ConstBufferRef compressStream(const CompressionConfig &config, nbostream &stream, vespalib::DataBuffer & compressed_data) { - using compression::compress; + using vespalib::compression::compress; vespalib::ConstBufferRef buf(stream.c_str(), stream.size()); if (config.useCompression() && bigEnough(stream.size(), config)) { CompressionConfig::Type compressedType = compress(config, vespalib::ConstBufferRef(stream.c_str(), stream.size()), compressed_data, false); diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt index 2179b1307d3..8cb148abe25 100644 --- a/document/src/vespa/document/util/CMakeLists.txt +++ b/document/src/vespa/document/util/CMakeLists.txt @@ -2,9 +2,6 @@ vespa_add_library(document_util OBJECT SOURCES bytebuffer.cpp - compressor.cpp - lz4compressor.cpp - zstdcompressor.cpp printable.cpp serializable.cpp stringutil.cpp diff --git a/document/src/vespa/document/util/compressionconfig.h b/document/src/vespa/document/util/compressionconfig.h deleted file mode 100644 index 413bdc1fb3b..00000000000 --- a/document/src/vespa/document/util/compressionconfig.h +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <cmath> -#include <cstdint> -#include <cstddef> - -namespace document { - -struct CompressionConfig { - enum Type { - NONE = 0, - HISTORIC_1 = 1, - HISTORIC_2 = 2, - HISTORIC_3 = 3, - HISTORIC_4 = 4, - UNCOMPRESSABLE = 5, - LZ4 = 6, - ZSTD = 7 - }; - - CompressionConfig() - : type(NONE), compressionLevel(0), threshold(90), minSize(0) {} - CompressionConfig(Type t) - : type(t), compressionLevel(9), threshold(90), minSize(0) {} - - CompressionConfig(Type t, uint8_t level, uint8_t minRes) - : type(t), compressionLevel(level), threshold(minRes), minSize(0) {} - - CompressionConfig(Type t, uint8_t lvl, uint8_t minRes, size_t minSz) - : type(t), compressionLevel(lvl), threshold(minRes), minSize(minSz) {} - - bool operator==(const CompressionConfig& o) const { - return (type == o.type - && compressionLevel == o.compressionLevel - && threshold == o.threshold); - } - bool operator!=(const CompressionConfig& o) const { - return !operator==(o); - } - - static Type toType(uint32_t val) { - switch (val) { - case 1: return HISTORIC_1; - case 2: return HISTORIC_2; - case 3: return HISTORIC_3; - case 4: return HISTORIC_4; - case 5: return UNCOMPRESSABLE; - case 6: return LZ4; - case 7: return ZSTD; - default: return NONE; - } - } - static bool isCompressed(Type type) { - return (type != CompressionConfig::NONE && - type != CompressionConfig::UNCOMPRESSABLE); - } - bool useCompression() const { return isCompressed(type); } - - Type type; - uint8_t compressionLevel; - uint8_t threshold; - size_t minSize; -}; - -class CompressionInfo -{ -public: - CompressionInfo(size_t uncompressedSize) - : _uncompressedSize(uncompressedSize), _compressedSize(uncompressedSize) { } - CompressionInfo(size_t uncompressedSize, size_t compressedSize) - : _uncompressedSize(uncompressedSize), _compressedSize(compressedSize) { } - size_t getUncompressedSize() const { return _uncompressedSize; } - size_t getCompressedSize() const { return _compressedSize; } - double getCompressionRatio() const { return _uncompressedSize/_compressedSize; } -private: - size_t _uncompressedSize; - size_t _compressedSize; -}; - -inline CompressionInfo operator + (const CompressionInfo & a, const CompressionInfo & b) -{ - return CompressionInfo(a.getUncompressedSize() + b.getUncompressedSize(), a.getCompressedSize() + b.getCompressedSize()); -} - -} - - diff --git a/document/src/vespa/document/util/compressor.cpp b/document/src/vespa/document/util/compressor.cpp deleted file mode 100644 index cd45017dd69..00000000000 --- a/document/src/vespa/document/util/compressor.cpp +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "lz4compressor.h" -#include "zstdcompressor.h" -#include <vespa/vespalib/util/memory.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/data/databuffer.h> - -using vespalib::alloc::Alloc; -using vespalib::ConstBufferRef; -using vespalib::DataBuffer; -using vespalib::make_string; - -namespace document::compression { - -CompressionConfig::Type -compress(ICompressor & compressor, const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest) -{ - CompressionConfig::Type type(CompressionConfig::NONE); - dest.ensureFree(compressor.adjustProcessLen(0, org.size())); - size_t compressedSize(dest.getFreeLen()); - if (compressor.process(compression, org.c_str(), org.size(), dest.getFree(), compressedSize)) { - if (compressedSize < ((org.size() * compression.threshold)/100)) { - dest.moveFreeToData(compressedSize); - type = compression.type; - } - } - return type; -} - -CompressionConfig::Type -docompress(const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest) -{ - CompressionConfig::Type type(CompressionConfig::NONE); - switch (compression.type) { - case CompressionConfig::LZ4: - { - LZ4Compressor lz4; - type = compress(lz4, compression, org, dest); - } - break; - case CompressionConfig::ZSTD: - { - ZStdCompressor zstd; - type = compress(zstd, compression, org, dest); - } - break; - case CompressionConfig::NONE: - default: - break; - } - return type; -} - -CompressionConfig::Type -compress(const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap) -{ - CompressionConfig::Type type(CompressionConfig::NONE); - if (org.size() >= compression.minSize) { - type = docompress(compression, org, dest); - } - if (type == CompressionConfig::NONE) { - if (allowSwap) { - DataBuffer tmp(const_cast<char *>(org.c_str()), org.size()); - tmp.moveFreeToData(org.size()); - dest.swap(tmp); - } else { - dest.writeBytes(org.c_str(), org.size()); - } - } - return type; -} - - -void -decompress(ICompressor & decompressor, size_t uncompressedLen, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap) -{ - dest.ensureFree(uncompressedLen); - size_t realUncompressedLen(dest.getFreeLen()); - if ( ! decompressor.unprocess(org.c_str(), org.size(), dest.getFree(), realUncompressedLen) ) { - if ( uncompressedLen < realUncompressedLen) { - if (allowSwap) { - DataBuffer tmp(const_cast<char *>(org.c_str()), org.size()); - tmp.moveFreeToData(org.size()); - dest.swap(tmp); - } else { - dest.writeBytes(org.c_str(), org.size()); - } - } else { - throw std::runtime_error(make_string("unprocess failed had %" PRIu64 ", wanted %" PRId64 ", got %" PRIu64, - org.size(), uncompressedLen, realUncompressedLen)); - } - } else { - dest.moveFreeToData(realUncompressedLen); - } -} - -void -decompress(const CompressionConfig::Type & type, size_t uncompressedLen, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap) -{ - switch (type) { - case CompressionConfig::LZ4: - { - LZ4Compressor lz4; - decompress(lz4, uncompressedLen, org, dest, allowSwap); - } - break; - case CompressionConfig::ZSTD: - { - ZStdCompressor zstd; - decompress(zstd, uncompressedLen, org, dest, allowSwap); - } - break; - case CompressionConfig::NONE: - case CompressionConfig::UNCOMPRESSABLE: - if (allowSwap) { - DataBuffer tmp(const_cast<char *>(org.c_str()), org.size()); - tmp.moveFreeToData(org.size()); - dest.swap(tmp); - } else { - dest.writeBytes(org.c_str(), org.size()); - } - break; - default: - throw std::runtime_error(make_string("Unable to handle decompression of type '%d'", type)); - break; - } -} - -size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t payloadSize) { - if (type == CompressionConfig::LZ4) { - document::LZ4Compressor lz4; - return lz4.adjustProcessLen(0, payloadSize); - } else if (type == CompressionConfig::ZSTD) { - document::ZStdCompressor zstd; - return zstd.adjustProcessLen(0, payloadSize); - } - return payloadSize; -} - -} diff --git a/document/src/vespa/document/util/compressor.h b/document/src/vespa/document/util/compressor.h deleted file mode 100644 index a8d4803e038..00000000000 --- a/document/src/vespa/document/util/compressor.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "compressionconfig.h" -#include <vespa/vespalib/util/buffer.h> - -namespace vespalib { class DataBuffer; } - -namespace document { - -class ICompressor -{ -public: - virtual ~ICompressor() { } - virtual bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) = 0; - virtual bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) = 0; - virtual size_t adjustProcessLen(uint16_t options, size_t len) const = 0; -}; - -namespace compression { - -/** - * Will try to compress a buffer according to the config. If the criteria can not - * be met it will return NONE and dest will get the input buffer. - * @param compression is config for how to compress and what criteria to meet. - * @param org is the original input buffer. - * @param dest is the destination buffer. The compressed data will be appended unless allowSwap is true - * and it is not compressable. Then it will be swapped in. - * @param allowSwap will tell it the data must be appended or if it can be swapped in if it is uncompressable or config is NONE. - */ -CompressionConfig::Type compress(const CompressionConfig & compression, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap); - -/** - * Will try to decompress a buffer according to the config. - * be met it will return NONE and dest will get the input buffer. - * @param compression is the compression type used for the buffer. - * @param uncompressedLen is the length of the uncompressed data. - * @param org is the original input buffer. - * @param dest is the destination buffer. The decompressed data will be - * appended unless allowSwap is true and compression is NONE. - * Then it will be swapped in. - * @param allowSwap will tell it the data must be appended or if it can be swapped in if compression type is NONE. - */ -void decompress(const CompressionConfig::Type & compression, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap); - - -size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t uncompressedSize); - -} - -} diff --git a/document/src/vespa/document/util/lz4compressor.cpp b/document/src/vespa/document/util/lz4compressor.cpp deleted file mode 100644 index 04d68f394a1..00000000000 --- a/document/src/vespa/document/util/lz4compressor.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "lz4compressor.h" -#include <vespa/vespalib/util/alloc.h> -#include <lz4.h> -#include <lz4hc.h> -#include <cassert> - -using vespalib::alloc::Alloc; - -namespace document { - -size_t LZ4Compressor::adjustProcessLen(uint16_t, size_t len) const { return LZ4_compressBound(len); } - -bool -LZ4Compressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) -{ - const char * input(static_cast<const char *>(inputV)); - char * output(static_cast<char *>(outputV)); - int sz(-1); - int maxOutputLen = LZ4_compressBound(inputLen); - if (config.compressionLevel > 6) { - Alloc state = Alloc::alloc(LZ4_sizeofStateHC()); - sz = LZ4_compress_HC_extStateHC(state.get(), input, output, inputLen, maxOutputLen, config.compressionLevel); - } else { - Alloc state = Alloc::alloc(LZ4_sizeofState()); - sz = LZ4_compress_fast_extState(state.get(), input, output, inputLen, maxOutputLen, 1); - } - assert(sz != 0); - outputLenV = sz; - return (sz != 0); - -} - -bool -LZ4Compressor::unprocess(const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) -{ - const char * input(static_cast<const char *>(inputV)); - char * output(static_cast<char *>(outputV)); - int sz = LZ4_decompress_safe(input, output, inputLen, outputLenV); - assert(sz > 0); - outputLenV = sz; - return (sz > 0); -} - -} diff --git a/document/src/vespa/document/util/lz4compressor.h b/document/src/vespa/document/util/lz4compressor.h deleted file mode 100644 index 6dd8f8fd1cb..00000000000 --- a/document/src/vespa/document/util/lz4compressor.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "compressor.h" - -namespace document { - -class LZ4Compressor : public ICompressor -{ -public: - bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) override; - bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) override; - size_t adjustProcessLen(uint16_t options, size_t len) const override; -}; - -} - diff --git a/document/src/vespa/document/util/zstdcompressor.cpp b/document/src/vespa/document/util/zstdcompressor.cpp deleted file mode 100644 index 74a17212a2e..00000000000 --- a/document/src/vespa/document/util/zstdcompressor.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "zstdcompressor.h" -#include <vespa/vespalib/util/alloc.h> -#include <vespa/vespalib/util/sync.h> -#include <zstd.h> -#include <vector> -#include <cassert> - -using vespalib::alloc::Alloc; - -namespace document { - -namespace { - -class CompressContext { -public: - CompressContext() : _ctx(ZSTD_createCCtx()) {} - ~CompressContext() { ZSTD_freeCCtx(_ctx); } - ZSTD_CCtx * get() { return _ctx; } -private: - ZSTD_CCtx * _ctx; -}; -class DecompressContext { -public: - DecompressContext() : _ctx(ZSTD_createDCtx()) {} - ~DecompressContext() { ZSTD_freeDCtx(_ctx); } - ZSTD_DCtx * get() { return _ctx; } -private: - ZSTD_DCtx * _ctx; -}; - -thread_local std::unique_ptr<CompressContext> _tlCompressState; -thread_local std::unique_ptr<DecompressContext> _tlDecompressState; - -} - -size_t ZStdCompressor::adjustProcessLen(uint16_t, size_t len) const { return ZSTD_compressBound(len); } - -bool -ZStdCompressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) -{ - size_t maxOutputLen = ZSTD_compressBound(inputLen); - if ( ! _tlCompressState) { - _tlCompressState = std::make_unique<CompressContext>(); - } - size_t sz = ZSTD_compressCCtx(_tlCompressState->get(), outputV, maxOutputLen, inputV, inputLen, config.compressionLevel); - assert( ! ZSTD_isError(sz) ); - outputLenV = sz; - return ! ZSTD_isError(sz); -} - -bool -ZStdCompressor::unprocess(const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) -{ - if ( ! _tlDecompressState) { - _tlDecompressState = std::make_unique<DecompressContext>(); - } - size_t sz = ZSTD_decompressDCtx(_tlDecompressState->get(), outputV, outputLenV, inputV, inputLen); - assert( ! ZSTD_isError(sz) ); - outputLenV = sz; - return ! ZSTD_isError(sz); -} - -} diff --git a/document/src/vespa/document/util/zstdcompressor.h b/document/src/vespa/document/util/zstdcompressor.h deleted file mode 100644 index ba12ed2594c..00000000000 --- a/document/src/vespa/document/util/zstdcompressor.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "compressor.h" - -namespace document { - -class ZStdCompressor : public ICompressor -{ -public: - bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) override; - bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) override; - size_t adjustProcessLen(uint16_t options, size_t len) const override; -}; - -} - |