diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2017-08-28 14:54:05 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-08-28 14:54:05 +0200 |
commit | 10751c1edd819792d499131cd549fd972accc2de (patch) | |
tree | a1ad230e5ea4f204d4b6aaf1e929afee82420c55 | |
parent | 622361fbcb6314ca23d18f64248d4f5f84230803 (diff) | |
parent | e3f443548f596443e8ebfca7eec8bc5211f7055a (diff) |
Merge pull request #3230 from vespa-engine/balder/move-databuffer-and-compression-to-vespalib
Balder/move databuffer and compression to vespalib
90 files changed, 239 insertions, 208 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 003ba9a5261..b03c0cb752f 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -21,7 +21,6 @@ import com.yahoo.slime.BinaryFormat; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.data.access.Inspector; -import com.yahoo.text.Utf8String; import com.yahoo.vespa.config.search.DispatchConfig; import java.util.Iterator; diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index e9694390b4b..ca2ee029c87 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -8,10 +8,6 @@ vespa_define_module( config_cloudconfig vespaeval - EXTERNAL_DEPENDS - lz4 - zstd - LIBS src/vespa/document src/vespa/document/annotation diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index 44cff2c80de..c3f88ac3696 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -16,6 +16,7 @@ #include <fcntl.h> using vespalib::nbostream; +using vespalib::compression::CompressionConfig; using namespace document::config_builder; diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp index 508e6f237cf..1e8fd9ec470 100644 --- a/document/src/tests/repo/documenttyperepo_test.cpp +++ b/document/src/tests/repo/documenttyperepo_test.cpp @@ -10,7 +10,6 @@ #include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> -#include <stdlib.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/testapp.h> @@ -25,6 +24,7 @@ using std::vector; using vespalib::Identifiable; using vespalib::IllegalArgumentException; using vespalib::string; +using vespalib::compression::CompressionConfig; using namespace document::config_builder; using namespace document; diff --git a/document/src/tests/serialization/.gitignore b/document/src/tests/serialization/.gitignore index 9f5bc440533..7e94d2757ae 100644 --- a/document/src/tests/serialization/.gitignore +++ b/document/src/tests/serialization/.gitignore @@ -2,5 +2,4 @@ .depend Makefile document_annotationserializer_test_app -document_compression_test_app document_vespadocumentserializer_test_app diff --git a/document/src/tests/serialization/CMakeLists.txt b/document/src/tests/serialization/CMakeLists.txt index 5b23bffad26..e1ce43b12d4 100644 --- a/document/src/tests/serialization/CMakeLists.txt +++ b/document/src/tests/serialization/CMakeLists.txt @@ -17,12 +17,3 @@ vespa_add_executable(document_annotationserializer_test_app TEST document_documentconfig ) vespa_add_test(NAME document_annotationserializer_test_app COMMAND document_annotationserializer_test_app) -vespa_add_executable(document_compression_test_app TEST - SOURCES - compression_test.cpp - DEPENDS - document - AFTER - document_documentconfig -) -vespa_add_test(NAME document_compression_test_app COMMAND document_compression_test_app) diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index d09012a4e4b..9da20e5a84c 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -55,6 +55,7 @@ using vespalib::tensor::Tensor; using vespalib::tensor::TensorBuilder; using vespalib::tensor::TensorCells; using vespalib::tensor::TensorDimensions; +using vespalib::compression::CompressionConfig; using namespace document; using std::string; using std::vector; diff --git a/document/src/vespa/document/datatype/structdatatype.h b/document/src/vespa/document/datatype/structdatatype.h index f6c77f18f47..0f1c58316c9 100644 --- a/document/src/vespa/document/datatype/structdatatype.h +++ b/document/src/vespa/document/datatype/structdatatype.h @@ -11,15 +11,16 @@ #include <vespa/document/datatype/structureddatatype.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <memory> namespace document { class StructDataType final : public StructuredDataType { public: - typedef std::unique_ptr<StructDataType> UP; - typedef std::shared_ptr<StructDataType> SP; + using UP = std::unique_ptr<StructDataType>; + using SP = std::shared_ptr<StructDataType>; + using CompressionConfig = vespalib::compression::CompressionConfig; StructDataType(); StructDataType(const vespalib::stringref &name); diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index 76216d75f1b..5dfd8eff891 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -2,7 +2,7 @@ #include "serializablearray.h" #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/util/bytebuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/data/databuffer.h> #include <algorithm> @@ -181,7 +181,7 @@ SerializableArray::clear(int id) void SerializableArray::deCompress() // throw (DeserializeException) { - using document::compression::decompress; + using vespalib::compression::decompress; // will only do this once LOG_ASSERT(_compSerData); @@ -239,7 +239,7 @@ void SerializableArray::assign(EntryMap & entries, } } -CompressionInfo +vespalib::compression::CompressionInfo SerializableArray::getCompressionInfo() const { return CompressionInfo(_uncompressedLength, _compSerData->getRemaining()); } diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h index 2d12c1191a6..2f7d65938aa 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.h +++ b/document/src/vespa/document/fieldvalue/serializablearray.h @@ -16,7 +16,7 @@ #pragma once -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/objects/cloneable.h> #include <vespa/vespalib/util/buffer.h> #include <vespa/vespalib/util/memory.h> @@ -107,6 +107,8 @@ public: using CP = vespalib::CloneablePtr<SerializableArray>; using UP = std::unique_ptr<SerializableArray>; using ByteBufferUP = std::unique_ptr<ByteBuffer>; + using CompressionConfig = vespalib::compression::CompressionConfig; + using CompressionInfo = vespalib::compression::CompressionInfo; SerializableArray(); virtual ~SerializableArray(); diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index c0ae342fd34..0a05ae60600 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -21,6 +21,7 @@ using std::vector; using vespalib::nbostream; using vespalib::nbostream_longlivedbuf; using vespalib::make_string; +using vespalib::compression::CompressionConfig; using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index 6ab6f71ce56..bb6956cf012 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -52,7 +52,9 @@ private: mutable bool _hasChanged; public: - typedef std::unique_ptr<StructFieldValue> UP; + using UP = std::unique_ptr<StructFieldValue>; + using CompressionConfig = vespalib::compression::CompressionConfig; + StructFieldValue(const DataType &type); ~StructFieldValue(); void swap(StructFieldValue & rhs); diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index 870e88e5036..6bfae246c10 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -30,6 +30,7 @@ using vespalib::hash_map; using vespalib::make_string; using vespalib::string; using vespalib::stringref; +using vespalib::compression::CompressionConfig; namespace document { diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index bee1c258e4a..2b45e8a298c 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -38,6 +38,7 @@ using vespalib::asciistream; using vespalib::nbostream; using vespalib::Memory; using vespalib::stringref; +using vespalib::compression::CompressionConfig; namespace document { diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 2534690b014..6c9e95a9dd6 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -27,7 +27,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> using std::make_pair; using std::pair; @@ -36,6 +36,7 @@ using vespalib::nbostream; using vespalib::stringref; using vespalib::string; using vespalib::slime::BinaryFormat; +using vespalib::compression::CompressionConfig; namespace document { @@ -275,7 +276,7 @@ vespalib::ConstBufferRef compressStream(const CompressionConfig &config, nbostream &stream, vespalib::DataBuffer & compressed_data) { - using compression::compress; + using vespalib::compression::compress; vespalib::ConstBufferRef buf(stream.c_str(), stream.size()); if (config.useCompression() && bigEnough(stream.size(), config)) { CompressionConfig::Type compressedType = compress(config, vespalib::ConstBufferRef(stream.c_str(), stream.size()), compressed_data, false); diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt index 2179b1307d3..8cb148abe25 100644 --- a/document/src/vespa/document/util/CMakeLists.txt +++ b/document/src/vespa/document/util/CMakeLists.txt @@ -2,9 +2,6 @@ vespa_add_library(document_util OBJECT SOURCES bytebuffer.cpp - compressor.cpp - lz4compressor.cpp - zstdcompressor.cpp printable.cpp serializable.cpp stringutil.cpp diff --git a/documentapi/src/tests/policies/testframe.cpp b/documentapi/src/tests/policies/testframe.cpp index d231672da84..877e3164a8c 100644 --- a/documentapi/src/tests/policies/testframe.cpp +++ b/documentapi/src/tests/policies/testframe.cpp @@ -7,6 +7,7 @@ #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/simplereply.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/log/log.h> LOG_SETUP(".testframe"); diff --git a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp index d942049192a..26a745df148 100644 --- a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp +++ b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp @@ -9,6 +9,7 @@ #include <vespa/messagebus/destinationsession.h> #include <vespa/messagebus/protocolset.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/signalhandler.h> #include <vespa/vespalib/util/slaveproc.h> diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index a17c19804c5..82c7b22c8c7 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -6,7 +6,7 @@ #include <vespa/searchlib/util/rawbuf.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/searchlib/common/transport.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/log/log.h> @@ -20,6 +20,8 @@ using vespalib::stringref; using vespalib::ConstBufferRef; using vespalib::DataBuffer; using vespalib::Memory; +using vespalib::compression::CompressionConfig; + namespace proton { @@ -377,7 +379,7 @@ TEST("requireThatSlimeInterfaceWorksFine") { } void -verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t orgSize, size_t compressedSize, +verifyReply(size_t count, CompressionConfig::Type encoding, size_t orgSize, size_t compressedSize, FRT_RPCRequest *request) { FRT_Values &ret = *request->GetReturn(); EXPECT_EQUAL(encoding, ret[0]._intval8); @@ -386,7 +388,8 @@ verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t org DataBuffer uncompressed; ConstBufferRef blob(ret[2]._data._buf, ret[2]._data._len); - compression::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, blob, uncompressed, false); + vespalib::compression::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, + blob, uncompressed, false); EXPECT_EQUAL(orgSize, uncompressed.getDataLen()); vespalib::Slime summaries; @@ -396,8 +399,8 @@ verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t org void verifyRPC(size_t count, - document::CompressionConfig::Type requestCompression, size_t requestSize, size_t requestBlobSize, - document::CompressionConfig::Type replyCompression, size_t replySize, size_t replyBlobSize) { + CompressionConfig::Type requestCompression, size_t requestSize, size_t requestBlobSize, + CompressionConfig::Type replyCompression, size_t replySize, size_t replyBlobSize) { Server server; vespalib::Slime slimeRequest = createSlimeRequestLarger(count); vespalib::SimpleBuffer buf; @@ -406,8 +409,9 @@ verifyRPC(size_t count, CompressionConfig config(requestCompression, 9, 100); DataBuffer compressed(const_cast<char *>(buf.get().data), buf.get().size); - CompressionConfig::Type type = compression::compress(config, ConstBufferRef(buf.get().data, buf.get().size), - compressed, true); + CompressionConfig::Type type = vespalib::compression::compress(config, + ConstBufferRef(buf.get().data, buf.get().size), + compressed, true); EXPECT_EQUAL(type, requestCompression); FRT_RPCRequest *request = new FRT_RPCRequest(); @@ -424,9 +428,9 @@ verifyRPC(size_t count, } TEST("requireThatRPCInterfaceWorks") { - verifyRPC(1, document::CompressionConfig::NONE, 55, 55, document::CompressionConfig::NONE, 38, 38); - verifyRPC(100, document::CompressionConfig::NONE, 2631, 2631, document::CompressionConfig::LZ4, 1426, 46); - verifyRPC(100, document::CompressionConfig::LZ4, 2631, 69, document::CompressionConfig::LZ4, 1426, 46); + verifyRPC(1, CompressionConfig::NONE, 55, 55, CompressionConfig::NONE, 38, 38); + verifyRPC(100, CompressionConfig::NONE, 2631, 2631, CompressionConfig::LZ4, 1426, 46); + verifyRPC(100, CompressionConfig::LZ4, 2631, 69, CompressionConfig::LZ4, 1426, 46); } } diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp index fc0a57ff519..b6a1133d01d 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp @@ -19,7 +19,7 @@ LOG_SETUP(".fdispatch"); using search::fs4transport::FS4PersistentPacketStreamer; using vespa::config::search::core::FdispatchrcConfig; using vespa::config::search::core::internal::InternalFdispatchrcType; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; char FastS_VersionTag[] = V_TAG; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index e65209bf526..79d82108ee8 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -13,7 +13,6 @@ #include <vespa/searchsummary/docsummary/docsumconfig.h> #include <vespa/vespalib/util/exceptions.h> #include <sstream> -#include <future> #include <vespa/log/log.h> LOG_SETUP(".proton.docsummary.summarymanager"); @@ -26,6 +25,8 @@ using namespace vespa::config::search::summary; using namespace vespa::config::search; using vespalib::make_string; using vespalib::IllegalArgumentException; +using vespalib::compression::CompressionConfig; + using search::DocumentStore; using search::IDocumentStore; using search::LogDocumentStore; @@ -160,13 +161,13 @@ SummaryManager::createSummarySetup(const SummaryConfig & summaryCfg, namespace { template<typename T> -document::CompressionConfig +CompressionConfig deriveCompression(const T & config) { - document::CompressionConfig compression; + CompressionConfig compression; if (config.type == T::LZ4) { - compression.type = document::CompressionConfig::LZ4; + compression.type = CompressionConfig::LZ4; } else if (config.type == T::ZSTD) { - compression.type = document::CompressionConfig::ZSTD; + compression.type = CompressionConfig::ZSTD; } compression.compressionLevel = config.level; return compression; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index ed350453da2..d95b0fd44d1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -48,7 +48,7 @@ using search::index::SchemaBuilder; using search::transactionlog::DomainStats; using vespa::config::search::core::ProtonConfig; using vespa::config::search::core::internal::InternalProtonType; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index 2e8c985fc0a..0595cb5d983 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -9,7 +9,7 @@ LOG_SETUP(".proton.server.rtchooks"); using namespace vespalib; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace { diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp index 9a37e0ae495..79928f96d7a 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsum_by_slime.h" -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/common/packets.h> #include <vespa/fnet/frt/rpcrequest.h> @@ -23,7 +23,7 @@ using vespalib::slime::ArrayTraverser; using vespalib::SimpleBuffer; using vespalib::DataBuffer; using vespalib::ConstBufferRef; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace { @@ -99,8 +99,8 @@ DocsumByRPC::DocsumByRPC(DocsumBySlime & slimeDocsumServer) : void DocsumByRPC::getDocsums(FRT_RPCRequest & req) { - using document::compression::decompress; - using document::compression::compress; + using vespalib::compression::decompress; + using vespalib::compression::compress; FRT_Values &arg = *req.GetParams(); uint8_t encoding = arg[0]._intval8; uint32_t uncompressedSize = arg[1]._intval32; diff --git a/searchlib/src/apps/docstore/create-idx-from-dat.cpp b/searchlib/src/apps/docstore/create-idx-from-dat.cpp index ac3b34a3b41..6124b36de8d 100644 --- a/searchlib/src/apps/docstore/create-idx-from-dat.cpp +++ b/searchlib/src/apps/docstore/create-idx-from-dat.cpp @@ -39,7 +39,7 @@ bool tryDecode(size_t chunks, size_t offset, const char * p, size_t sz, size_t n } bool validUncompressed(const char * n, size_t offset) { - return (n[1] == document::CompressionConfig::NONE) && + return (n[1] == vespalib::compression::CompressionConfig::NONE) && (n[2] == 0) && (n[3] == 0) && (n[4] == 0) && diff --git a/searchlib/src/tests/common/packets/packets_test.cpp b/searchlib/src/tests/common/packets/packets_test.cpp index 7504e1c1570..cf0e858a014 100644 --- a/searchlib/src/tests/common/packets/packets_test.cpp +++ b/searchlib/src/tests/common/packets/packets_test.cpp @@ -8,6 +8,8 @@ #include <vespa/fnet/controlpacket.h> using namespace search::fs4transport; +using vespalib::compression::CompressionConfig; + // ---------------------------------------------------------------------------- // @@ -524,7 +526,7 @@ TEST("test pre serializing packets with compression") { EXPECT_EQUAL(src->GetLength(), decoded->GetLength()); FS4PersistentPacketStreamer::Instance.SetCompressionLimit(100); FS4Packet_PreSerialized serialized(*src); - EXPECT_EQUAL(218u | (document::CompressionConfig::LZ4 << 24), serialized.GetPCODE()); + EXPECT_EQUAL(218u | (CompressionConfig::LZ4 << 24), serialized.GetPCODE()); EXPECT_GREATER_EQUAL(321u, serialized.GetLength()); FNET_Packet::UP decoded2(testEncodeDecode(serialized)); EXPECT_EQUAL(500u, decoded2->GetLength()); diff --git a/searchlib/src/tests/docstore/chunk/chunk_test.cpp b/searchlib/src/tests/docstore/chunk/chunk_test.cpp index 4687f45acde..84ac877c54d 100644 --- a/searchlib/src/tests/docstore/chunk/chunk_test.cpp +++ b/searchlib/src/tests/docstore/chunk/chunk_test.cpp @@ -12,7 +12,7 @@ LOG_SETUP("chunk_test"); using namespace search; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; TEST("require that Chunk obey limits") { diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index e3e4a1432d1..e8c2173a87f 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -5,6 +5,7 @@ #include <vespa/document/repo/documenttyperepo.h> using namespace search; +using CompressionConfig = vespalib::compression::CompressionConfig; document::DocumentTypeRepo repo; @@ -43,7 +44,7 @@ struct NullDataStore : IDataStore { }; TEST_FFF("require that uncache docstore lookups are counted", - DocumentStore::Config(document::CompressionConfig::NONE, 0, 0), + DocumentStore::Config(CompressionConfig::NONE, 0, 0), NullDataStore(), DocumentStore(f1, f2)) { EXPECT_EQUAL(0u, f3.getCacheStats().misses); @@ -52,7 +53,7 @@ TEST_FFF("require that uncache docstore lookups are counted", } TEST_FFF("require that cached docstore lookups are counted", - DocumentStore::Config(document::CompressionConfig::NONE, 100000, 100), + DocumentStore::Config(CompressionConfig::NONE, 100000, 100), NullDataStore(), DocumentStore(f1, f2)) { EXPECT_EQUAL(0u, f3.getCacheStats().misses); diff --git a/searchlib/src/tests/docstore/document_store/visitcache_test.cpp b/searchlib/src/tests/docstore/document_store/visitcache_test.cpp index 70e0c7f01fb..d5d95097c66 100644 --- a/searchlib/src/tests/docstore/document_store/visitcache_test.cpp +++ b/searchlib/src/tests/docstore/document_store/visitcache_test.cpp @@ -56,11 +56,12 @@ void verifyAB(const BlobSet & a) { using B=vespalib::ConstBufferRef; TEST("require that BlobSet can be built") { + using CompressionConfig = vespalib::compression::CompressionConfig; BlobSet a; a.append(7, B("aaaaaa",6)); a.append(9, B("bbbbb",5)); verifyAB(a); - document::CompressionConfig cfg(document::CompressionConfig::LZ4); + CompressionConfig cfg(CompressionConfig::LZ4); CompressedBlobSet ca(cfg, a); BlobSet b = ca.getBlobSet(); verifyAB(b); diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index 944d0a543f5..1c7053500c7 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -24,7 +24,7 @@ using document::Document; using document::DocumentId; using document::DocumentType; using document::DocumentTypeRepo; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; using vespalib::asciistream; using index::DummyFileHeaderContext; diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index 6d75c6365f9..598913a3222 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -113,6 +113,7 @@ struct ReadFixture : public FixtureBase { struct WriteFixture : public FixtureBase { WriteableFileChunk chunk; + using CompressionConfig = vespalib::compression::CompressionConfig; WriteFixture(const vespalib::string &baseName, uint32_t docIdLimit, @@ -124,7 +125,7 @@ struct WriteFixture : public FixtureBase { baseName, serialNum, docIdLimit, - WriteableFileChunk::Config(document::CompressionConfig(), 0x1000), + WriteableFileChunk::Config(CompressionConfig(), 0x1000), tuneFile, fileHeaderCtx, &bucketizer, diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index 82a158fd53e..ed6afb06681 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -15,7 +15,6 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <iomanip> -#include <iostream> using document::BucketId; using namespace search::docstore; @@ -36,7 +35,7 @@ public: using namespace search; using namespace search::docstore; using search::index::DummyFileHeaderContext; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace { diff --git a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp index 3faf9395ec5..e9514c1d385 100644 --- a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp +++ b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp @@ -4,17 +4,17 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/document/base/documentid.h> -#include <vespa/log/log.h> #include <vespa/searchlib/docstore/storebybucket.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/log/log.h> LOG_SETUP("store_by_bucket_test"); using namespace search::docstore; using document::BucketId; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; vespalib::string createPayload(BucketId b) { diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp index 0848d9996c3..4a69e0a827d 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp @@ -2,9 +2,9 @@ #include "attribute_header.h" #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> -namespace search { -namespace attribute { +namespace search::attribute { namespace { @@ -168,5 +168,4 @@ AttributeHeader::hasWeightedSetType() const return _collectionType.isWeightedSet(); } -} // namespace search::attribute -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.h b/searchlib/src/vespa/searchlib/attribute/readerbase.h index 88081906f71..358038f9ba2 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/searchlib/util/fileutil.h> +#include <cassert> namespace search { diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index e50ebec105b..aa39b798b7a 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -5,10 +5,9 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> -namespace search { - -namespace bitcompression { +namespace search::bitcompression { using vespalib::nbostream; @@ -157,17 +156,12 @@ DecodeContext64Base::checkPointRead(nbostream &in) (void) in; } -} // namespace bitcompression - - namespace { vespalib::string noFeatures = "NoFeatures"; } -namespace bitcompression { - template <bool bigEndian> void FeatureDecodeContext<bigEndian>:: @@ -486,6 +480,4 @@ template class FeatureEncodeContext<true>; template class FeatureEncodeContext<false>; -} // namespace bitcompression - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 667334bf64e..a0b64e2ef94 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -4,7 +4,7 @@ #include "packets.h" #include "sortdata.h" #include <vespa/searchlib/util/rawbuf.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/databuffer.h> @@ -12,7 +12,6 @@ #include <vespa/log/log.h> LOG_SETUP(".searchlib.common.fs4packets"); -using document::CompressionConfig; using vespalib::ConstBufferRef; using vespalib::make_string; using vespalib::stringref; @@ -151,7 +150,7 @@ FS4PersistentPacketStreamer::Decode(FNET_DataBuffer *src, uint32_t plen, uint32_ uint32_t uncompressed_size = src->ReadInt32(); ConstBufferRef org(src->GetData(), plen - sizeof(uint32_t)); vespalib::DataBuffer uncompressed(uncompressed_size); - document::compression::decompress(compressionType, uncompressed_size, org, uncompressed, false); + vespalib::compression::decompress(compressionType, uncompressed_size, org, uncompressed, false); FNET_DataBuffer buf(uncompressed.getData(), uncompressed.getDataLen()); decodePacket(packet, buf, uncompressed_size, pcode); src->DataToDead(plen - sizeof(uint32_t)); @@ -192,7 +191,7 @@ FS4PersistentPacketStreamer::Encode(FNET_Packet *packet, uint32_t chid, FNET_Dat CompressionConfig config(_compressionType, _compressionLevel, 90); ConstBufferRef org(dst->GetData() + packet_start + header_len, body_len); vespalib::DataBuffer compressed(org.size()); - CompressionConfig::Type r = document::compression::compress(config, org, compressed, false); + CompressionConfig::Type r = vespalib::compression::compress(config, org, compressed, false); if (r != CompressionConfig::NONE) { dst->DataToFree(body_len + header_len); // sizeof(data + header + uncompressed_size) - sizeof(uint32_t) @@ -455,7 +454,7 @@ FS4Packet_PreSerialized::FS4Packet_PreSerialized(FNET_Packet & packet) 90); ConstBufferRef org(tmp.GetData(), tmp.GetDataLen()); vespalib::DataBuffer compressed(org.size()); - _compressionType = document::compression::compress(config, org, compressed, false); + _compressionType = vespalib::compression::compress(config, org, compressed, false); if (_compressionType != CompressionConfig::NONE) { _data.WriteInt32Fast(body_len); _data.WriteBytes(compressed.getData(), compressed.getDataLen()); @@ -1285,18 +1284,18 @@ FS4Packet_QUERYX::Encode(FNET_DataBuffer *dst) void FS4Packet::throwPropertieDecodeError(size_t i) { - throw vespalib::IllegalArgumentException(vespalib::make_string("Failed decoding properties[%ld]", i)); + throw vespalib::IllegalArgumentException(make_string("Failed decoding properties[%ld]", i)); } void FS4Packet::throwUnsupportedFeatures(uint32_t features, uint32_t set) { - throw vespalib::UnderflowException(vespalib::make_string("Unsupported features(%x), supported set(%x)", features, set)); + throw vespalib::UnderflowException(make_string("Unsupported features(%x), supported set(%x)", features, set)); } void FS4Packet::throwNotEnoughData(FNET_DataBuffer & buf, uint32_t left, uint32_t needed, const char * text) { (void) buf; - throw vespalib::UnderflowException(vespalib::make_string("Failed decoding packet of type %d. Only %d bytes left, needed %d from '%s'", GetPCODE(), left, needed, text)); + throw vespalib::UnderflowException(make_string("Failed decoding packet of type %d. Only %d bytes left, needed %d from '%s'", GetPCODE(), left, needed, text)); } #define VERIFY_LEN(needed, text) \ @@ -1436,7 +1435,7 @@ FS4Packet_QUERYX::toString(uint32_t indent) const } s += make_string("%*s sortspec : %s\n", indent, "", _sortSpec.c_str()); s += make_string("%*s groupspec : (%d bytes)\n", indent, "", (int)_groupSpec.size()); - s += make_string("%*s sessionId : (%d bytes)\n", indent, "", (int)_sessionId.size()); + s += make_string("%*s sessionId : (%d bytes) %s\n", indent, "", (int)_sessionId.size(), _sessionId.c_str()); s += make_string("%*s location : %s\n", indent, "", _location.c_str()); s += make_string("%*s timeout : %d\n", indent, "", _timeout); s += make_string("%*s stackitems : %d\n", indent, "", _numStackItems); diff --git a/searchlib/src/vespa/searchlib/common/packets.h b/searchlib/src/vespa/searchlib/common/packets.h index d9bc4d50462..52130c57374 100644 --- a/searchlib/src/vespa/searchlib/common/packets.h +++ b/searchlib/src/vespa/searchlib/common/packets.h @@ -9,7 +9,7 @@ #include <vespa/fnet/packet.h> #include <vespa/fnet/databuffer.h> #include <vespa/document/base/globalid.h> -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/util/memory.h> #include <vespa/fastos/timestamp.h> #include <vector> @@ -110,10 +110,11 @@ public: class FS4PersistentPacketStreamer : public FNET_IPacketStreamer { FS4PersistentPacketStreamer(const FS4PersistentPacketStreamer &); FS4PersistentPacketStreamer& operator=(const FS4PersistentPacketStreamer &); + using CompressionConfig = vespalib::compression::CompressionConfig; unsigned int _compressionLimit; unsigned int _compressionLevel; - document::CompressionConfig::Type _compressionType; + CompressionConfig::Type _compressionType; protected: bool _conservative; // Set to true if out of sync should mark the // stream as broken. @@ -139,8 +140,8 @@ public: void SetConservativeMode(bool cons) { _conservative = cons; } void SetCompressionLimit(unsigned int limit) { _compressionLimit = limit; } void SetCompressionLevel(unsigned int level) { _compressionLevel = level; } - void SetCompressionType(document::CompressionConfig::Type compressionType) { _compressionType = compressionType; } - document::CompressionConfig::Type getCompressionType() const { return _compressionType; } + void SetCompressionType(CompressionConfig::Type compressionType) { _compressionType = compressionType; } + CompressionConfig::Type getCompressionType() const { return _compressionType; } uint32_t getCompressionLimit() const { return _compressionLimit; } uint32_t getCompressionLevel() const { return _compressionLevel; } }; @@ -243,9 +244,10 @@ public: bool Decode(FNET_DataBuffer *src, uint32_t len) override; vespalib::string toString(uint32_t indent) const override; private: - uint32_t _pcode; - document::CompressionConfig::Type _compressionType; - FNET_DataBuffer _data; + using CompressionConfig = vespalib::compression::CompressionConfig; + uint32_t _pcode; + CompressionConfig::Type _compressionType; + FNET_DataBuffer _data; }; class FS4Packet_Shared : public FS4Packet diff --git a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp b/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp index a55ec9494dc..0324f00f63c 100644 --- a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp @@ -1,18 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - #include "checkpointfile.h" #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/common/fileheadercontext.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".diskindex.checkpointfile"); using vespalib::getLastErrorString; -namespace search { - -namespace diskindex { +namespace search::diskindex { using common::FileHeaderContext; @@ -182,6 +180,4 @@ CheckPointFile::readHeader() } -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp index 179fda8689b..ba467501fba 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp @@ -43,12 +43,12 @@ Chunk::hasRoom(size_t len) const } size_t -Chunk::getMaxPackSize(const document::CompressionConfig & compression) const { +Chunk::getMaxPackSize(const CompressionConfig & compression) const { return _format->getMaxPackSize(compression); } void -Chunk::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const document::CompressionConfig & compression) +Chunk::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const CompressionConfig & compression) { _lastSerial = lastSerial; _format->pack(_lastSerial, compressed, compression); diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.h b/searchlib/src/vespa/searchlib/docstore/chunk.h index bdf0d9793b2..cd551de1a9e 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.h +++ b/searchlib/src/vespa/searchlib/docstore/chunk.h @@ -4,7 +4,7 @@ #include <vespa/searchlib/util/memoryusage.h> #include <vespa/vespalib/util/buffer.h> -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <memory> #include <vector> @@ -60,7 +60,8 @@ private: class Chunk { public: - typedef std::unique_ptr<Chunk> UP; + using UP = std::unique_ptr<Chunk>; + using CompressionConfig = vespalib::compression::CompressionConfig; class Config { public: Config(size_t maxBytes) : _maxBytes(maxBytes) { } @@ -93,8 +94,8 @@ public: size_t size() const; const LidList & getLids() const { return _lids; } LidList getUniqueLids() const; - size_t getMaxPackSize(const document::CompressionConfig & compression) const; - void pack(uint64_t lastSerial, vespalib::DataBuffer & buffer, const document::CompressionConfig & compression); + size_t getMaxPackSize(const CompressionConfig & compression) const; + void pack(uint64_t lastSerial, vespalib::DataBuffer & buffer, const CompressionConfig & compression); uint64_t getLastSerial() const { return _lastSerial; } uint32_t getId() const { return _id; } bool validSerial() const { return getLastSerial() != static_cast<uint64_t>(-1l); } diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index 204c7b07acb..37381cfa3f6 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -1,17 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "chunkformats.h" -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/util/stringfmt.h> namespace search { using vespalib::make_string; using vespalib::Exception; -using document::compression::compress; -using document::compression::decompress; -using document::compression::computeMaxCompressedsize; -using document::CompressionConfig; +using vespalib::compression::compress; +using vespalib::compression::decompress; +using vespalib::compression::computeMaxCompressedsize; +using vespalib::compression::CompressionConfig; ChunkException::ChunkException(const vespalib::stringref & msg, const vespalib::stringref & location) : Exception(make_string("Illegal chunk: %s", msg.c_str()), location) diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.h b/searchlib/src/vespa/searchlib/docstore/chunkformat.h index 5f1e3d852a9..9f9580f1f1d 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.h +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/exception.h> @@ -20,7 +20,8 @@ class ChunkFormat { public: virtual ~ChunkFormat(); - typedef std::unique_ptr<ChunkFormat> UP; + using UP = std::unique_ptr<ChunkFormat>; + using CompressionConfig = vespalib::compression::CompressionConfig; vespalib::nbostream & getBuffer() { return _dataBuf; } const vespalib::nbostream & getBuffer() const { return _dataBuf; } @@ -30,7 +31,7 @@ public: * @param compressed The buffer where the serialized data shall be placed. * @param compression What kind of compression shall be employed. */ - void pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const document::CompressionConfig & compression); + void pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const CompressionConfig & compression); /** * Will deserialize and create a representation of the uncompressed data. * param buffer Pointer to the serialized data @@ -44,7 +45,7 @@ public: * @param compression Compression config to be used. * @return maximum number of bytes a packet can take in serialized form. */ - size_t getMaxPackSize(const document::CompressionConfig & compression) const; + size_t getMaxPackSize(const CompressionConfig & compression) const; protected: /** * Constructor used when deserializing diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.h b/searchlib/src/vespa/searchlib/docstore/compacter.h index 264cce8d3aa..362896487aa 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.h +++ b/searchlib/src/vespa/searchlib/docstore/compacter.h @@ -33,7 +33,7 @@ private: **/ class BucketCompacter : public IWriteData, public StoreByBucket::IWrite { - using CompressionConfig = document::CompressionConfig; + using CompressionConfig = vespalib::compression::CompressionConfig; using ThreadExecutor = vespalib::ThreadExecutor; public: using FileId = FileChunk::FileId; diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp index a13fffdaa24..bf59614a297 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp @@ -6,12 +6,12 @@ #include "ibucketizer.h" #include <vespa/vespalib/stllike/cache.hpp> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> using document::DocumentTypeRepo; -using document::CompressionConfig; -using document::compression::compress; -using document::compression::decompress; +using vespalib::compression::CompressionConfig; +using vespalib::compression::compress; +using vespalib::compression::decompress; namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h index 45dfcb35e37..4ba5c27cd07 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h @@ -26,25 +26,26 @@ class DocumentStore : public IDocumentStore public: class Config { public: + using CompressionConfig = vespalib::compression::CompressionConfig; Config() : - _compression(document::CompressionConfig::LZ4, 9, 70), + _compression(CompressionConfig::LZ4, 9, 70), _maxCacheBytes(1000000000), _initialCacheEntries(0), _allowVisitCaching(false) { } - Config(const document::CompressionConfig & compression, size_t maxCacheBytes, size_t initialCacheEntries) : - _compression((maxCacheBytes != 0) ? compression : document::CompressionConfig::NONE), + Config(const CompressionConfig & compression, size_t maxCacheBytes, size_t initialCacheEntries) : + _compression((maxCacheBytes != 0) ? compression : CompressionConfig::NONE), _maxCacheBytes(maxCacheBytes), _initialCacheEntries(initialCacheEntries), _allowVisitCaching(false) { } - const document::CompressionConfig & getCompression() const { return _compression; } + const CompressionConfig & getCompression() const { return _compression; } size_t getMaxCacheBytes() const { return _maxCacheBytes; } size_t getInitialCacheEntries() const { return _initialCacheEntries; } bool allowVisitCaching() const { return _allowVisitCaching; } Config & allowVisitCaching(bool allow) { _allowVisitCaching = allow; return *this; } private: - document::CompressionConfig _compression; + CompressionConfig _compression; size_t _maxCacheBytes; size_t _initialCacheEntries; bool _allowVisitCaching; diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index f271e6f320b..2a748a302c6 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -5,6 +5,7 @@ #include "summaryexceptions.h" #include "randreaders.h" #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index 0ca658ba803..080e6f80503 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -5,7 +5,7 @@ #include "idatastore.h" #include "lid_info.h" #include "writeablefilechunk.h" -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/searchlib/common/rcuvector.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/transactionlog/syncproxy.h> @@ -33,7 +33,7 @@ private: public: using NameIdSet = std::set<NameId>; using LockGuard = vespalib::LockGuard; - using CompressionConfig = document::CompressionConfig; + using CompressionConfig = vespalib::compression::CompressionConfig; class Config { public: Config() diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.h b/searchlib/src/vespa/searchlib/docstore/storebybucket.h index f93826027f5..ac1f6fbe007 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.h +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.h @@ -23,9 +23,9 @@ class StoreByBucket using MemoryDataStore = vespalib::MemoryDataStore; using ThreadExecutor = vespalib::ThreadExecutor; using ConstBufferRef = vespalib::ConstBufferRef; - using CompressionConfig = document::CompressionConfig; + using CompressionConfig = vespalib::compression::CompressionConfig; public: - StoreByBucket(vespalib::MemoryDataStore & backingMemory, const document::CompressionConfig & compression); + StoreByBucket(vespalib::MemoryDataStore & backingMemory, const CompressionConfig & compression); StoreByBucket(MemoryDataStore & backingMemory, ThreadExecutor & executor, const CompressionConfig & compression); StoreByBucket(StoreByBucket &&) = default; ~StoreByBucket(); diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index 22a16947b67..b3fd236d73d 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -5,7 +5,7 @@ #include <vespa/vespalib/stllike/cache.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <algorithm> namespace search::docstore { @@ -74,7 +74,7 @@ BlobSet::get(uint32_t lid) const } CompressedBlobSet::CompressedBlobSet() : - _compression(document::CompressionConfig::Type::LZ4), + _compression(CompressionConfig::Type::LZ4), _positions(), _buffer() { @@ -83,7 +83,7 @@ CompressedBlobSet::CompressedBlobSet() : CompressedBlobSet::~CompressedBlobSet() { } -CompressedBlobSet::CompressedBlobSet(const document::CompressionConfig &compression, const BlobSet & uncompressed) : +CompressedBlobSet::CompressedBlobSet(const CompressionConfig &compression, const BlobSet & uncompressed) : _compression(compression.type), _positions(uncompressed.getPositions()), _buffer() @@ -91,7 +91,7 @@ CompressedBlobSet::CompressedBlobSet(const document::CompressionConfig &compress if ( ! _positions.empty() ) { DataBuffer compressed; ConstBufferRef org = uncompressed.getBuffer(); - _compression = document::compression::compress(compression, org, compressed, false); + _compression = vespalib::compression::compress(compression, org, compressed, false); _buffer.resize(compressed.getDataLen()); memcpy(_buffer, compressed.getData(), compressed.getDataLen()); } @@ -100,7 +100,7 @@ CompressedBlobSet::CompressedBlobSet(const document::CompressionConfig &compress BlobSet CompressedBlobSet::getBlobSet() const { - using document::compression::decompress; + using vespalib::compression::decompress; // These are frequent lage allocations that are to expensive to mmap. DataBuffer uncompressed(0, 1, Alloc::alloc(0, 16 * MemoryAllocator::HUGEPAGE_SIZE)); if ( ! _positions.empty() ) { @@ -145,7 +145,7 @@ VisitCache::BackingStore::read(const KeySet &key, CompressedBlobSet &blobs) cons return ! blobs.empty(); } -VisitCache::VisitCache(IDataStore &store, size_t cacheSize, const document::CompressionConfig &compression) : +VisitCache::VisitCache(IDataStore &store, size_t cacheSize, const CompressionConfig &compression) : _store(store, compression), _cache(std::make_unique<Cache>(_store, cacheSize)) { diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h index 44ee5542a72..a89620b7bde 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.h +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h @@ -72,8 +72,9 @@ private: **/ class CompressedBlobSet { public: + using CompressionConfig = vespalib::compression::CompressionConfig; CompressedBlobSet(); - CompressedBlobSet(const document::CompressionConfig &compression, const BlobSet & uncompressed); + CompressedBlobSet(const CompressionConfig &compression, const BlobSet & uncompressed); CompressedBlobSet(CompressedBlobSet && rhs) = default; CompressedBlobSet & operator=(CompressedBlobSet && rhs) = default; CompressedBlobSet(const CompressedBlobSet & rhs) = default; @@ -83,7 +84,7 @@ public: bool empty() const { return _positions.empty(); } BlobSet getBlobSet() const; private: - document::CompressionConfig::Type _compression; + CompressionConfig::Type _compression; BlobSet::Positions _positions; vespalib::MallocPtr _buffer; }; @@ -95,7 +96,8 @@ private: **/ class VisitCache { public: - VisitCache(IDataStore &store, size_t cacheSize, const document::CompressionConfig &compression); + using CompressionConfig = vespalib::compression::CompressionConfig; + VisitCache(IDataStore &store, size_t cacheSize, const CompressionConfig &compression); CompressedBlobSet read(const IDocumentStore::LidVector & keys) const; void remove(uint32_t key); @@ -111,17 +113,17 @@ private: */ class BackingStore { public: - BackingStore(IDataStore &store, const document::CompressionConfig &compression) : + BackingStore(IDataStore &store, const CompressionConfig &compression) : _backingStore(store), _compression(compression) { } bool read(const KeySet &key, CompressedBlobSet &blobs) const; void write(const KeySet &, const CompressedBlobSet &) { } void erase(const KeySet &) { } - const document::CompressionConfig &getCompression() const { return _compression; } + const CompressionConfig &getCompression() const { return _compression; } private: IDataStore &_backingStore; - const document::CompressionConfig _compression; + const CompressionConfig _compression; }; using CacheParams = vespalib::CacheParam< diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index c6b93a59ae8..5cf5c646148 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/closuretask.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/objects/nbostream.h> diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index 86415e0cb40..2b21f12a314 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -22,20 +22,21 @@ public: class Config { public: + using CompressionConfig = vespalib::compression::CompressionConfig; Config() - : _compression(document::CompressionConfig::LZ4, 9, 60), + : _compression(CompressionConfig::LZ4, 9, 60), _maxChunkBytes(0x10000) { } - Config(const document::CompressionConfig &compression, size_t maxChunkBytes) + Config(const CompressionConfig &compression, size_t maxChunkBytes) : _compression(compression), _maxChunkBytes(maxChunkBytes) { } - const document::CompressionConfig & getCompression() const { return _compression; } + const CompressionConfig & getCompression() const { return _compression; } size_t getMaxChunkBytes() const { return _maxChunkBytes; } private: - document::CompressionConfig _compression; + CompressionConfig _compression; size_t _maxChunkBytes; }; diff --git a/searchlib/src/vespa/searchlib/grouping/hyperloglog.h b/searchlib/src/vespa/searchlib/grouping/hyperloglog.h index 753b81d92f2..931b832c76d 100644 --- a/searchlib/src/vespa/searchlib/grouping/hyperloglog.h +++ b/searchlib/src/vespa/searchlib/grouping/hyperloglog.h @@ -3,8 +3,8 @@ #pragma once #include "sketch.h" -#include <vespa/document/util/compressionconfig.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressionconfig.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/objects/deserializer.h> #include <vespa/vespalib/objects/serializer.h> diff --git a/searchlib/src/vespa/searchlib/grouping/sketch.h b/searchlib/src/vespa/searchlib/grouping/sketch.h index bb7db02d81c..317c1bfef9d 100644 --- a/searchlib/src/vespa/searchlib/grouping/sketch.h +++ b/searchlib/src/vespa/searchlib/grouping/sketch.h @@ -2,8 +2,8 @@ #pragma once -#include <vespa/document/util/compressionconfig.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressionconfig.h> +#include <vespa/vespalib/util/compressor.h> #include <lz4.h> #include <vespa/searchlib/common/identifiable.h> #include <vespa/vespalib/data/databuffer.h> @@ -205,13 +205,13 @@ deserialize(vespalib::Deserializer &is) { template <int BucketBits, typename HashT> uint32_t NormalSketch<BucketBits, HashT>:: compress_buckets_into(char *buffer, uint32_t size) const { - document::CompressionConfig config(document::CompressionConfig::LZ4, 9, 9); + vespalib::compression::CompressionConfig config(vespalib::compression::CompressionConfig::LZ4, 9, 9); vespalib::ConstBufferRef org(&bucket[0], BUCKET_COUNT); vespalib::DataBuffer compress_buffer(buffer, size); - document::CompressionConfig::Type r = - document::compression::compress(config, org, compress_buffer, false); + vespalib::compression::CompressionConfig::Type r = + vespalib::compression::compress(config, org, compress_buffer, false); assert(compress_buffer.getDead() == buffer); - if (r == document::CompressionConfig::LZ4) { + if (r == vespalib::compression::CompressionConfig::LZ4) { assert(compress_buffer.getDataLen() < BUCKET_COUNT); return compress_buffer.getDataLen(); } else { @@ -228,7 +228,8 @@ decompress_buckets_from(char *buffer, uint32_t size) { } else { vespalib::ConstBufferRef compressed(buffer, size); vespalib::DataBuffer uncompressed(reinterpret_cast<char *>(&bucket[0]), BUCKET_COUNT); - document::compression::decompress(document::CompressionConfig::LZ4, BUCKET_COUNT, compressed, uncompressed, false); + vespalib::compression::decompress(vespalib::compression::CompressionConfig::LZ4, BUCKET_COUNT, + compressed, uncompressed, false); } } template <int BucketBits, typename HashT> diff --git a/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp b/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp index 6c4155b035e..e5e582b673b 100644 --- a/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp +++ b/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/util/fileheadertk.h> #include <vespa/vespalib/util/host_name.h> +#include <cassert> #include <unistd.h> namespace search::index { diff --git a/searchlib/src/vespa/searchlib/util/fileutil.cpp b/searchlib/src/vespa/searchlib/util/fileutil.cpp index c1d34f66b0b..674608bdda1 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.cpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.cpp @@ -4,6 +4,7 @@ #include "filesizecalculator.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/guard.h> +#include <cassert> #include <fcntl.h> #include <sys/mman.h> #include <sys/stat.h> @@ -18,7 +19,6 @@ using vespalib::FileDescriptor; using vespalib::getLastErrorString; namespace search { - namespace fileutil { LoadedMmap::LoadedMmap(const vespalib::string &fileName) @@ -177,4 +177,5 @@ FileWriterBase::write(const void *buf, size_t sz) { } return numWritten; } + } diff --git a/staging_vespalib/CMakeLists.txt b/staging_vespalib/CMakeLists.txt index 5a169416775..d59529df828 100644 --- a/staging_vespalib/CMakeLists.txt +++ b/staging_vespalib/CMakeLists.txt @@ -13,7 +13,6 @@ vespa_define_module( src/tests/bits src/tests/clock src/tests/crc - src/tests/databuffer src/tests/directio src/tests/encoding/base64 src/tests/fileheader diff --git a/staging_vespalib/src/tests/databuffer/.gitignore b/staging_vespalib/src/tests/databuffer/.gitignore deleted file mode 100644 index e7b0e69c372..00000000000 --- a/staging_vespalib/src/tests/databuffer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_databuffer_test_app diff --git a/staging_vespalib/src/tests/databuffer/CMakeLists.txt b/staging_vespalib/src/tests/databuffer/CMakeLists.txt deleted file mode 100644 index 0c96a8cdf3e..00000000000 --- a/staging_vespalib/src/tests/databuffer/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_databuffer_test_app TEST - SOURCES - databuffer_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_databuffer_test_app COMMAND staging_vespalib_databuffer_test_app) diff --git a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp b/staging_vespalib/src/tests/fileheader/fileheader_test.cpp index b895b92db4c..f6f9c19b48c 100644 --- a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp +++ b/staging_vespalib/src/tests/fileheader/fileheader_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/fastos/file.h> #include <iostream> diff --git a/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt index 3fbece6211d..8ca70d2a63d 100644 --- a/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(staging_vespalib_vespalib_data OBJECT SOURCES - databuffer.cpp fileheader.cpp DEPENDS ) diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp b/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp index 3cd8190d894..a9e131b0c63 100644 --- a/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp +++ b/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "fileheader.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.h b/staging_vespalib/src/vespa/vespalib/data/fileheader.h index 157703ef0a8..e4449a0c36a 100644 --- a/staging_vespalib/src/vespa/vespalib/data/fileheader.h +++ b/staging_vespalib/src/vespa/vespalib/data/fileheader.h @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "databuffer.h" #include <vespa/vespalib/util/exception.h> #include <map> @@ -9,6 +8,7 @@ class FastOS_FileInterface; namespace vespalib { +class DataBuffer; class asciistream; /** diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index 23c713d41f9..2c00dbe2f37 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -17,3 +17,11 @@ mbus_content_node_max_pending_count int default=0 mbus_distributor_node_max_pending_size int default=0 mbus_content_node_max_pending_size int default=0 +# Minimum size of packets to compress (0 means no compression) +mbus.compress.limit int default=1024 + +## Compression level for packets +mbus.compress.level int default=3 + +## Compression type for packets. +mbus.compress.type enum {NONE, LZ4, ZSTD} default=LZ4 diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index aac52550fa0..ae8f3290fef 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -8,6 +8,7 @@ #include <vespa/documentapi/messagebus/messages/wrongdistributionreply.h> #include <vespa/storageapi/message/state.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> @@ -362,8 +363,7 @@ void CommunicationManager::onClose() } void -CommunicationManager::configureMessageBusLimits( - const CommunicationManagerConfig& cfg) +CommunicationManager::configureMessageBusLimits(const CommunicationManagerConfig& cfg) { const bool isDist(_component.getNodeType() == lib::NodeType::DISTRIBUTOR); auto& mbus(_mbus->getMessageBus()); diff --git a/storageserver/src/tests/storageservertest.cpp b/storageserver/src/tests/storageservertest.cpp index f3595afe1e7..21b190d9e76 100644 --- a/storageserver/src/tests/storageservertest.cpp +++ b/storageserver/src/tests/storageservertest.cpp @@ -6,6 +6,7 @@ #include <vespa/document/base/testdocman.h> #include <vespa/documentapi/documentapi.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h> #include <vespa/messagebus/staticthrottlepolicy.h> #include <vespa/messagebus/testlib/slobrok.h> @@ -20,8 +21,8 @@ #include <vespa/storageserver/app/distributorprocess.h> #include <vespa/storageserver/app/memfileservicelayerprocess.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/fnet/frt/supervisor.h> #include <sys/time.h> -#include <fstream> #include <vespa/log/log.h> LOG_SETUP(".storageservertest"); diff --git a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp index ce55c3a87f8..940d142db2f 100644 --- a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp +++ b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp @@ -5,17 +5,17 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/objects/nbostream.h> +using vespalib::compression::CompressionConfig; using vespalib::nbostream; namespace vdslib { -MutableDocumentList::MutableDocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, uint32_t bufferSize, bool keepexisting) +MutableDocumentList::MutableDocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, + uint32_t bufferSize, bool keepexisting) : DocumentList(repo, buffer, bufferSize, keepexisting) { } -MutableDocumentList::MutableDocumentList(const DocumentList& source, - char* buffer, - uint32_t bufferSize) +MutableDocumentList::MutableDocumentList(const DocumentList& source, char* buffer, uint32_t bufferSize) : DocumentList(source, buffer, bufferSize) { } @@ -41,8 +41,7 @@ MutableDocumentList::addOperationList(const OperationList& opl) } bool -MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, - bool addBody) +MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, bool addBody) { uint32_t freePos = _freePtr - _buffer; @@ -69,9 +68,7 @@ MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, entry.bodyLen = bodySize; entry.flags = 0; - if (doc.getType().getFieldsType().getCompressionConfig().type - != document::CompressionConfig::NONE) - { + if (doc.getType().getFieldsType().getCompressionConfig().type != CompressionConfig::NONE) { entry.flags |= MetaEntry::COMPRESSED; } @@ -91,8 +88,7 @@ MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, } bool -MutableDocumentList::addUpdate(const document::DocumentUpdate& update, - Timestamp ts) +MutableDocumentList::addUpdate(const document::DocumentUpdate& update, Timestamp ts) { vespalib::nbostream os; update.serialize42(os); diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp index ec7c1c623c9..d9d8d0c4056 100644 --- a/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp +++ b/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp @@ -3,6 +3,8 @@ #include "mynetwork.h" #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/sendproxy.h> +#include <vespa/messagebus/network/oosmanager.h> + class MyServiceAddress : public mbus::IServiceAddress { private: diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 112f67f3a70..3ac7b388ec4 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -6,6 +6,7 @@ vespa_define_module( EXTERNAL_DEPENDS lz4 + zstd APPS src/apps/make_fixture_macros @@ -22,6 +23,8 @@ vespa_define_module( src/tests/closure src/tests/component src/tests/compress + src/tests/compression + src/tests/data/databuffer src/tests/data/input_reader src/tests/data/lz4_encode_decode src/tests/data/memory_input diff --git a/vespalib/src/tests/compression/.gitignore b/vespalib/src/tests/compression/.gitignore new file mode 100644 index 00000000000..60ffd040a47 --- /dev/null +++ b/vespalib/src/tests/compression/.gitignore @@ -0,0 +1,4 @@ +*_test +.depend +Makefile +vespalib_compression_test_app diff --git a/vespalib/src/tests/compression/CMakeLists.txt b/vespalib/src/tests/compression/CMakeLists.txt new file mode 100644 index 00000000000..c5738733218 --- /dev/null +++ b/vespalib/src/tests/compression/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_compression_test_app TEST + SOURCES + compression_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_compression_test_app COMMAND vespalib_compression_test_app) diff --git a/document/src/tests/serialization/compression_test.cpp b/vespalib/src/tests/compression/compression_test.cpp index 6574b4ac34c..01cfe0af223 100644 --- a/document/src/tests/serialization/compression_test.cpp +++ b/vespalib/src/tests/compression/compression_test.cpp @@ -2,15 +2,14 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/log/log.h> LOG_SETUP("compression_test"); -using namespace document; -using namespace document::compression; using namespace vespalib; +using namespace vespalib::compression; static vespalib::string _G_compressableText("AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" diff --git a/vespalib/src/tests/data/databuffer/.gitignore b/vespalib/src/tests/data/databuffer/.gitignore new file mode 100644 index 00000000000..f144796c66a --- /dev/null +++ b/vespalib/src/tests/data/databuffer/.gitignore @@ -0,0 +1 @@ +vespalib_data_databuffer_test_app diff --git a/vespalib/src/tests/data/databuffer/CMakeLists.txt b/vespalib/src/tests/data/databuffer/CMakeLists.txt new file mode 100644 index 00000000000..f1c6c7c1862 --- /dev/null +++ b/vespalib/src/tests/data/databuffer/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_data_databuffer_test_app TEST + SOURCES + databuffer_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_data_databuffer_test_app COMMAND vespalib_data_databuffer_test_app) diff --git a/staging_vespalib/src/tests/databuffer/databuffer_test.cpp b/vespalib/src/tests/data/databuffer/databuffer_test.cpp index f440ca1e15c..f440ca1e15c 100644 --- a/staging_vespalib/src/tests/databuffer/databuffer_test.cpp +++ b/vespalib/src/tests/data/databuffer/databuffer_test.cpp diff --git a/vespalib/src/vespa/vespalib/component/version.cpp b/vespalib/src/vespa/vespalib/component/version.cpp index 3fab6d6f130..af38a675de8 100644 --- a/vespalib/src/vespa/vespalib/component/version.cpp +++ b/vespalib/src/vespa/vespalib/component/version.cpp @@ -7,7 +7,6 @@ namespace vespalib { - Version::Version(int major, int minor, int micro, const string & qualifier) : _major(major), _minor(minor), @@ -89,7 +88,7 @@ Version::Version(const string & versionString) _qualifier(), _stringValue(versionString) { - if (versionString != "") { + if ( ! versionString.empty()) { stringref r(versionString.c_str(), versionString.size()); stringref::size_type dot(r.find('.')); stringref majorS(r.substr(0, dot)); diff --git a/vespalib/src/vespa/vespalib/data/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/CMakeLists.txt index 29c8055a0c0..3a94e00ae33 100644 --- a/vespalib/src/vespa/vespalib/data/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/data/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vespalib_vespalib_data OBJECT SOURCES + databuffer.cpp input.cpp input_reader.cpp lz4_input_decoder.cpp diff --git a/staging_vespalib/src/vespa/vespalib/data/databuffer.cpp b/vespalib/src/vespa/vespalib/data/databuffer.cpp index 529e475e987..5558a371836 100644 --- a/staging_vespalib/src/vespa/vespalib/data/databuffer.cpp +++ b/vespalib/src/vespa/vespalib/data/databuffer.cpp @@ -33,6 +33,7 @@ DataBuffer::DataBuffer(size_t len, size_t alignment, const Alloc & initial) } } +DataBuffer::~DataBuffer() = default; void DataBuffer::moveFreeToData(size_t len) diff --git a/staging_vespalib/src/vespa/vespalib/data/databuffer.h b/vespalib/src/vespa/vespalib/data/databuffer.h index f7707a3ea56..28524f373b2 100644 --- a/staging_vespalib/src/vespa/vespalib/data/databuffer.h +++ b/vespalib/src/vespa/vespalib/data/databuffer.h @@ -81,6 +81,8 @@ public: _buffer(Alloc::alloc(0)) { } + ~DataBuffer(); + /** * @return a pointer to the dead part of this buffer. **/ diff --git a/vespalib/src/vespa/vespalib/data/memory.h b/vespalib/src/vespa/vespalib/data/memory.h index 74024549a0b..eae7c8d2f23 100644 --- a/vespalib/src/vespa/vespalib/data/memory.h +++ b/vespalib/src/vespa/vespalib/data/memory.h @@ -25,6 +25,7 @@ struct Memory Memory(const vespalib::stringref &str_ref) : data(str_ref.data()), size(str_ref.size()) {} vespalib::string make_string() const; + vespalib::stringref make_stringref() const { return stringref(data, size); } bool operator == (const Memory &rhs) const { return ((size == rhs.size) && ((data == rhs.data) || diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 310e1dde68d..6d08c3b1126 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT classname.cpp closuretask.cpp compress.cpp + compressor.cpp dual_merge_director.cpp error.cpp exception.cpp @@ -25,6 +26,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT host_name.cpp joinable.cpp left_right_heap.cpp + lz4compressor.cpp md5.c printable.cpp priority_queue.cpp @@ -47,5 +49,6 @@ vespa_add_library(vespalib_vespalib_util OBJECT threadstackexecutorbase.cpp time_tracker.cpp valgrind.cpp + zstdcompressor.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/util/compress.cpp b/vespalib/src/vespa/vespalib/util/compress.cpp index bf32fcfe593..617b632e0bf 100644 --- a/vespalib/src/vespa/vespalib/util/compress.cpp +++ b/vespalib/src/vespa/vespalib/util/compress.cpp @@ -4,8 +4,7 @@ #include "stringfmt.h" #include "exceptions.h" -namespace vespalib { -namespace compress { +namespace vespalib::compress { size_t Integer::compressedPositiveLength(uint64_t n) { @@ -84,4 +83,3 @@ size_t Integer::compress(int64_t n, void *destination) } } -} diff --git a/document/src/vespa/document/util/compressionconfig.h b/vespalib/src/vespa/vespalib/util/compressionconfig.h index 413bdc1fb3b..bb54a74ea41 100644 --- a/document/src/vespa/document/util/compressionconfig.h +++ b/vespalib/src/vespa/vespalib/util/compressionconfig.h @@ -4,8 +4,9 @@ #include <cmath> #include <cstdint> #include <cstddef> +#include <cstring> -namespace document { +namespace vespalib::compression { struct CompressionConfig { enum Type { @@ -51,6 +52,14 @@ struct CompressionConfig { default: return NONE; } } + static Type toType(const char * val) { + if (strncasecmp(val, "lz4", 3) == 0) { + return LZ4; + } if (strncasecmp(val, "zstd", 4) == 0) { + return ZSTD; + } + return NONE; + } static bool isCompressed(Type type) { return (type != CompressionConfig::NONE && type != CompressionConfig::UNCOMPRESSABLE); diff --git a/document/src/vespa/document/util/compressor.cpp b/vespalib/src/vespa/vespalib/util/compressor.cpp index cd45017dd69..6853c8375fd 100644 --- a/document/src/vespa/document/util/compressor.cpp +++ b/vespalib/src/vespa/vespalib/util/compressor.cpp @@ -7,11 +7,8 @@ #include <vespa/vespalib/data/databuffer.h> using vespalib::alloc::Alloc; -using vespalib::ConstBufferRef; -using vespalib::DataBuffer; -using vespalib::make_string; -namespace document::compression { +namespace vespalib::compression { CompressionConfig::Type compress(ICompressor & compressor, const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest) @@ -129,10 +126,10 @@ decompress(const CompressionConfig::Type & type, size_t uncompressedLen, const C size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t payloadSize) { if (type == CompressionConfig::LZ4) { - document::LZ4Compressor lz4; + LZ4Compressor lz4; return lz4.adjustProcessLen(0, payloadSize); } else if (type == CompressionConfig::ZSTD) { - document::ZStdCompressor zstd; + ZStdCompressor zstd; return zstd.adjustProcessLen(0, payloadSize); } return payloadSize; diff --git a/document/src/vespa/document/util/compressor.h b/vespalib/src/vespa/vespalib/util/compressor.h index a8d4803e038..8f319a6735d 100644 --- a/document/src/vespa/document/util/compressor.h +++ b/vespalib/src/vespa/vespalib/util/compressor.h @@ -2,11 +2,11 @@ #pragma once #include "compressionconfig.h" -#include <vespa/vespalib/util/buffer.h> +#include "buffer.h" namespace vespalib { class DataBuffer; } -namespace document { +namespace vespalib::compression { class ICompressor { @@ -17,8 +17,6 @@ public: virtual size_t adjustProcessLen(uint16_t options, size_t len) const = 0; }; -namespace compression { - /** * Will try to compress a buffer according to the config. If the criteria can not * be met it will return NONE and dest will get the input buffer. @@ -43,9 +41,6 @@ CompressionConfig::Type compress(const CompressionConfig & compression, const ve */ void decompress(const CompressionConfig::Type & compression, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap); - size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t uncompressedSize); } - -} diff --git a/document/src/vespa/document/util/lz4compressor.cpp b/vespalib/src/vespa/vespalib/util/lz4compressor.cpp index 04d68f394a1..366609ce7bf 100644 --- a/document/src/vespa/document/util/lz4compressor.cpp +++ b/vespalib/src/vespa/vespalib/util/lz4compressor.cpp @@ -8,7 +8,7 @@ using vespalib::alloc::Alloc; -namespace document { +namespace vespalib::compression { size_t LZ4Compressor::adjustProcessLen(uint16_t, size_t len) const { return LZ4_compressBound(len); } diff --git a/document/src/vespa/document/util/lz4compressor.h b/vespalib/src/vespa/vespalib/util/lz4compressor.h index 6dd8f8fd1cb..558088a914c 100644 --- a/document/src/vespa/document/util/lz4compressor.h +++ b/vespalib/src/vespa/vespalib/util/lz4compressor.h @@ -3,7 +3,7 @@ #include "compressor.h" -namespace document { +namespace vespalib::compression { class LZ4Compressor : public ICompressor { diff --git a/document/src/vespa/document/util/zstdcompressor.cpp b/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp index 74a17212a2e..2f09abf4846 100644 --- a/document/src/vespa/document/util/zstdcompressor.cpp +++ b/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp @@ -9,7 +9,7 @@ using vespalib::alloc::Alloc; -namespace document { +namespace vespalib::compression { namespace { diff --git a/document/src/vespa/document/util/zstdcompressor.h b/vespalib/src/vespa/vespalib/util/zstdcompressor.h index ba12ed2594c..0d84b8e7ca4 100644 --- a/document/src/vespa/document/util/zstdcompressor.h +++ b/vespalib/src/vespa/vespalib/util/zstdcompressor.h @@ -3,7 +3,7 @@ #include "compressor.h" -namespace document { +namespace vespalib::compression { class ZStdCompressor : public ICompressor { |