aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2023-01-03 12:22:57 +0100
committerGitHub <noreply@github.com>2023-01-03 12:22:57 +0100
commit11eb110513640129fc617f67d2e3322fb0fb8674 (patch)
tree7b0b1af322fc72ddceda02c508457ae97ad266de
parent464a9a1be8f4c722ef878ccf9fec04a65735dd4f (diff)
parent6f85020f1b6896e9d79ae59f934be0dc4e7aa09a (diff)
Merge pull request #25368 from vespa-engine/balder/use-lockfree-atomic-compression-config
Enable having lock free atomic<CompresssionConfig> and make its use i…
-rw-r--r--searchlib/src/tests/docstore/document_store/document_store_test.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/docstore/chunk.cpp4
-rw-r--r--searchlib/src/vespa/searchlib/docstore/chunk.h4
-rw-r--r--searchlib/src/vespa/searchlib/docstore/chunkformat.cpp4
-rw-r--r--searchlib/src/vespa/searchlib/docstore/chunkformat.h4
-rw-r--r--searchlib/src/vespa/searchlib/docstore/compacter.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/docstore/compacter.h2
-rw-r--r--searchlib/src/vespa/searchlib/docstore/documentstore.cpp23
-rw-r--r--searchlib/src/vespa/searchlib/docstore/documentstore.h4
-rw-r--r--searchlib/src/vespa/searchlib/docstore/logdatastore.h2
-rw-r--r--searchlib/src/vespa/searchlib/docstore/storebybucket.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/docstore/storebybucket.h2
-rw-r--r--searchlib/src/vespa/searchlib/docstore/value.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/docstore/value.h2
-rw-r--r--searchlib/src/vespa/searchlib/docstore/visitcache.cpp15
-rw-r--r--searchlib/src/vespa/searchlib/docstore/visitcache.h13
-rw-r--r--searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h4
-rw-r--r--vespalib/src/tests/compression/compression_test.cpp5
-rw-r--r--vespalib/src/vespa/vespalib/util/compressionconfig.h18
-rw-r--r--vespalib/src/vespa/vespalib/util/compressor.cpp15
-rw-r--r--vespalib/src/vespa/vespalib/util/compressor.h23
-rw-r--r--vespalib/src/vespa/vespalib/util/lz4compressor.cpp2
-rw-r--r--vespalib/src/vespa/vespalib/util/lz4compressor.h2
-rw-r--r--vespalib/src/vespa/vespalib/util/zstdcompressor.cpp2
-rw-r--r--vespalib/src/vespa/vespalib/util/zstdcompressor.h2
25 files changed, 79 insertions, 81 deletions
diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp
index 5050592dc87..9fef6f57e85 100644
--- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp
+++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp
@@ -91,7 +91,7 @@ vespalib::stringref S1("this is a string long enough to be compressed and is jus
"Adding some repeatble sequences like aaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbbbbb to ensure compression"
"xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz xyz");
-Value createValue(vespalib::stringref s, const CompressionConfig & cfg) {
+Value createValue(vespalib::stringref s, CompressionConfig cfg) {
Value v(7);
vespalib::DataBuffer input;
input.writeBytes(s.data(), s.size());
diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp
index 2c0e47301d9..e83b190a7c0 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp
@@ -58,12 +58,12 @@ Chunk::hasRoom(size_t len) const
}
size_t
-Chunk::getMaxPackSize(const CompressionConfig & compression) const {
+Chunk::getMaxPackSize(CompressionConfig compression) const {
return _format->getMaxPackSize(compression);
}
void
-Chunk::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const CompressionConfig & compression)
+Chunk::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, CompressionConfig compression)
{
_lastSerial = lastSerial;
std::lock_guard guard(_lock);
diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.h b/searchlib/src/vespa/searchlib/docstore/chunk.h
index 158d754eb9a..93fc98311cd 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunk.h
+++ b/searchlib/src/vespa/searchlib/docstore/chunk.h
@@ -97,8 +97,8 @@ public:
size_t size() const;
const LidList & getLids() const { return _lids; }
LidList getUniqueLids() const;
- size_t getMaxPackSize(const CompressionConfig & compression) const;
- void pack(uint64_t lastSerial, vespalib::DataBuffer & buffer, const CompressionConfig & compression);
+ size_t getMaxPackSize(CompressionConfig compression) const;
+ void pack(uint64_t lastSerial, vespalib::DataBuffer & buffer, CompressionConfig compression);
uint64_t getLastSerial() const { return _lastSerial; }
uint32_t getId() const { return _id; }
bool validSerial() const { return getLastSerial() != static_cast<uint64_t>(-1l); }
diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
index 79a6742f9da..5b352b91c38 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
@@ -19,7 +19,7 @@ ChunkException::ChunkException(const vespalib::string & msg, vespalib::stringref
}
void
-ChunkFormat::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const CompressionConfig & compression)
+ChunkFormat::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, CompressionConfig compression)
{
vespalib::nbostream & os = _dataBuf;
os << lastSerial;
@@ -46,7 +46,7 @@ ChunkFormat::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const
}
size_t
-ChunkFormat::getMaxPackSize(const CompressionConfig & compression) const
+ChunkFormat::getMaxPackSize(CompressionConfig compression) const
{
const size_t OVERHEAD(0);
const size_t MINSIZE(1 + 1 + 4 + 4 + (includeSerializedSize() ? 4 : 0)); // version + type + real length + crc + lastserial
diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.h b/searchlib/src/vespa/searchlib/docstore/chunkformat.h
index 6fa13428b21..343f1dbb9c7 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunkformat.h
+++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.h
@@ -31,7 +31,7 @@ public:
* @param compressed The buffer where the serialized data shall be placed.
* @param compression What kind of compression shall be employed.
*/
- void pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const CompressionConfig & compression);
+ void pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, CompressionConfig compression);
/**
* Will deserialize and create a representation of the uncompressed data.
* param buffer Pointer to the serialized data
@@ -45,7 +45,7 @@ public:
* @param compression Compression config to be used.
* @return maximum number of bytes a packet can take in serialized form.
*/
- size_t getMaxPackSize(const CompressionConfig & compression) const;
+ size_t getMaxPackSize(CompressionConfig compression) const;
protected:
/**
* Constructor used when deserializing
diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.cpp b/searchlib/src/vespa/searchlib/docstore/compacter.cpp
index 33ed84ff4d0..04ff150c741 100644
--- a/searchlib/src/vespa/searchlib/docstore/compacter.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/compacter.cpp
@@ -23,7 +23,7 @@ Compacter::write(LockGuard guard, uint32_t chunkId, uint32_t lid, const void *bu
_ds.write(std::move(guard), fileId, lid, buffer, sz);
}
-BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, const CompressionConfig & compression, LogDataStore & ds,
+BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, CompressionConfig compression, LogDataStore & ds,
Executor & executor, const IBucketizer & bucketizer, FileId source, FileId destination) :
_unSignificantBucketBits((maxSignificantBucketBits > 8) ? (maxSignificantBucketBits - 8) : 0),
_sourceFileId(source),
diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.h b/searchlib/src/vespa/searchlib/docstore/compacter.h
index 9c5775c0c4a..0d7633b1699 100644
--- a/searchlib/src/vespa/searchlib/docstore/compacter.h
+++ b/searchlib/src/vespa/searchlib/docstore/compacter.h
@@ -35,7 +35,7 @@ class BucketCompacter : public IWriteData, public StoreByBucket::IWrite
using Executor = vespalib::Executor;
public:
using FileId = FileChunk::FileId;
- BucketCompacter(size_t maxSignificantBucketBits, const CompressionConfig & compression, LogDataStore & ds,
+ BucketCompacter(size_t maxSignificantBucketBits, CompressionConfig compression, LogDataStore & ds,
Executor & executor, const IBucketizer & bucketizer, FileId source, FileId destination);
void write(LockGuard guard, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) override ;
void write(BucketId bucketId, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) override;
diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
index 08d73719433..1e04b7c61db 100644
--- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
@@ -51,7 +51,7 @@ namespace docstore {
class BackingStore {
public:
- BackingStore(IDataStore &store, const CompressionConfig &compression) :
+ BackingStore(IDataStore &store, CompressionConfig compression) :
_backingStore(store),
_compression(compression)
{ }
@@ -60,11 +60,11 @@ public:
void visit(const IDocumentStore::LidVector &lids, const DocumentTypeRepo &repo, IDocumentVisitor &visitor) const;
void write(DocumentIdT, const Value &);
void erase(DocumentIdT) {}
- const CompressionConfig &getCompression() const { return _compression; }
- void reconfigure(const CompressionConfig &compression);
+ CompressionConfig getCompression() const { return _compression.load(std::memory_order_relaxed); }
+ void reconfigure(CompressionConfig compression);
private:
IDataStore &_backingStore;
- CompressionConfig _compression;
+ std::atomic<CompressionConfig> _compression;
};
void
@@ -80,7 +80,7 @@ BackingStore::read(DocumentIdT key, Value &value) const {
vespalib::DataBuffer buf(4_Ki);
ssize_t len = _backingStore.read(key, buf);
if (len > 0) {
- value.set(std::move(buf), len, _compression);
+ value.set(std::move(buf), len, getCompression());
found = true;
}
return found;
@@ -95,11 +95,8 @@ BackingStore::write(DocumentIdT lid, const Value & value)
}
void
-BackingStore::reconfigure(const CompressionConfig &compression) {
- if (compression != _compression) {
- // Need proper synchronization
- _compression = compression;
- }
+BackingStore::reconfigure(CompressionConfig compression) {
+ _compression.store(compression, std::memory_order_relaxed);
}
using CacheParams = vespalib::CacheParam<
@@ -291,14 +288,14 @@ class DocumentStore::WrapVisitor : public IDataStoreVisitor
{
Visitor &_visitor;
const DocumentTypeRepo &_repo;
- const CompressionConfig &_compression;
+ const CompressionConfig _compression;
IDocumentStore &_ds;
uint64_t _syncToken;
public:
void visit(uint32_t lid, const void *buffer, size_t sz) override;
- WrapVisitor(Visitor &visitor, const DocumentTypeRepo &repo, const CompressionConfig &compresion,
+ WrapVisitor(Visitor &visitor, const DocumentTypeRepo &repo, CompressionConfig compresion,
IDocumentStore &ds, uint64_t syncToken);
void rewrite(uint32_t lid, const document::Document &doc);
@@ -386,7 +383,7 @@ DocumentStore::WrapVisitor<Visitor>::visit(uint32_t lid, const void *buffer, siz
template <class Visitor>
DocumentStore::WrapVisitor<Visitor>::
-WrapVisitor(Visitor &visitor, const DocumentTypeRepo &repo, const CompressionConfig &compression,
+WrapVisitor(Visitor &visitor, const DocumentTypeRepo &repo, CompressionConfig compression,
IDocumentStore &ds, uint64_t syncToken)
: _visitor(visitor),
_repo(repo),
diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h
index 6d663bfb738..024b26b79c6 100644
--- a/searchlib/src/vespa/searchlib/docstore/documentstore.h
+++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h
@@ -31,13 +31,13 @@ public:
_initialCacheEntries(0),
_updateStrategy(INVALIDATE)
{ }
- Config(const CompressionConfig & compression, size_t maxCacheBytes, size_t initialCacheEntries) :
+ Config(CompressionConfig compression, size_t maxCacheBytes, size_t initialCacheEntries) :
_compression((maxCacheBytes != 0) ? compression : CompressionConfig::NONE),
_maxCacheBytes(maxCacheBytes),
_initialCacheEntries(initialCacheEntries),
_updateStrategy(INVALIDATE)
{ }
- const CompressionConfig & getCompression() const { return _compression; }
+ CompressionConfig getCompression() const { return _compression; }
size_t getMaxCacheBytes() const { return _maxCacheBytes; }
size_t getInitialCacheEntries() const { return _initialCacheEntries; }
Config & updateStrategy(UpdateStrategy strategy) { _updateStrategy = strategy; return *this; }
diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h
index 9366eb413d9..3289ceb19ba 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h
+++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h
@@ -58,7 +58,7 @@ public:
uint32_t getMaxNumLids() const { return _maxNumLids; }
bool crcOnReadDisabled() const { return _skipCrcOnRead; }
- const CompressionConfig & compactCompression() const { return _compactCompression; }
+ CompressionConfig compactCompression() const { return _compactCompression; }
const WriteableFileChunk::Config & getFileConfig() const { return _fileConfig; }
Config & disableCrcOnRead(bool v) { _skipCrcOnRead = v; return *this;}
diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp
index 651ff111f4e..6d3c39a51dc 100644
--- a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp
@@ -13,7 +13,7 @@ using document::BucketId;
using vespalib::CpuUsage;
using vespalib::makeLambdaTask;
-StoreByBucket::StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, const CompressionConfig & compression) noexcept
+StoreByBucket::StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, CompressionConfig compression) noexcept
: _chunkSerial(0),
_current(),
_where(),
diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.h b/searchlib/src/vespa/searchlib/docstore/storebybucket.h
index 1b5f9dc1204..dfe6199aa2e 100644
--- a/searchlib/src/vespa/searchlib/docstore/storebybucket.h
+++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.h
@@ -24,7 +24,7 @@ class StoreByBucket
using ConstBufferRef = vespalib::ConstBufferRef;
using CompressionConfig = vespalib::compression::CompressionConfig;
public:
- StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, const CompressionConfig & compression) noexcept;
+ StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, CompressionConfig compression) noexcept;
//TODO Putting the below move constructor into cpp file fails for some unknown reason. Needs to be resolved.
StoreByBucket(StoreByBucket &&) noexcept = default;
StoreByBucket(const StoreByBucket &) = delete;
diff --git a/searchlib/src/vespa/searchlib/docstore/value.cpp b/searchlib/src/vespa/searchlib/docstore/value.cpp
index bd4d75d944a..8ac43f7a2de 100644
--- a/searchlib/src/vespa/searchlib/docstore/value.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/value.cpp
@@ -55,7 +55,7 @@ compact(size_t sz, vespalib::alloc::Alloc buf) {
}
void
-Value::set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &compression) {
+Value::set(vespalib::DataBuffer &&buf, ssize_t len, CompressionConfig compression) {
assert(len < std::numeric_limits<uint32_t>::max());
//Underlying buffer must be identical to allow swap.
vespalib::DataBuffer compressed(buf.getData(), 0u);
diff --git a/searchlib/src/vespa/searchlib/docstore/value.h b/searchlib/src/vespa/searchlib/docstore/value.h
index e8f98f0a54e..9e98d4d4122 100644
--- a/searchlib/src/vespa/searchlib/docstore/value.h
+++ b/searchlib/src/vespa/searchlib/docstore/value.h
@@ -35,7 +35,7 @@ public:
* Compress buffer into temporary buffer and copy temporary buffer to
* value along with compression config.
*/
- void set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &compression);
+ void set(vespalib::DataBuffer &&buf, ssize_t len, CompressionConfig compression);
// Keep buffer uncompressed
void set(vespalib::DataBuffer &&buf, ssize_t len);
diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
index 041fb62d7de..60c08c281df 100644
--- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
@@ -84,7 +84,7 @@ CompressedBlobSet::CompressedBlobSet() :
CompressedBlobSet::~CompressedBlobSet() = default;
-CompressedBlobSet::CompressedBlobSet(const CompressionConfig &compression, const BlobSet & uncompressed) :
+CompressedBlobSet::CompressedBlobSet(CompressionConfig compression, const BlobSet & uncompressed) :
_compression(compression.type),
_positions(uncompressed.getPositions()),
_buffer()
@@ -144,27 +144,24 @@ bool
VisitCache::BackingStore::read(const KeySet &key, CompressedBlobSet &blobs) const {
VisitCollector collector;
_backingStore.read(key.getKeys(), collector);
- blobs = CompressedBlobSet(_compression, collector.getBlobSet());
+ blobs = CompressedBlobSet(_compression.load(std::memory_order_relaxed), collector.getBlobSet());
return ! blobs.empty();
}
void
-VisitCache::BackingStore::reconfigure(const CompressionConfig &compression) {
- if (compression != _compression) {
- // Need proper synchronization
- _compression = compression;
- }
+VisitCache::BackingStore::reconfigure(CompressionConfig compression) {
+ _compression.store(compression, std::memory_order_relaxed);
}
-VisitCache::VisitCache(IDataStore &store, size_t cacheSize, const CompressionConfig &compression) :
+VisitCache::VisitCache(IDataStore &store, size_t cacheSize, CompressionConfig compression) :
_store(store, compression),
_cache(std::make_unique<Cache>(_store, cacheSize))
{
}
void
-VisitCache::reconfigure(size_t cacheSize, const CompressionConfig &compression) {
+VisitCache::reconfigure(size_t cacheSize, CompressionConfig compression) {
_store.reconfigure(compression);
_cache->setCapacityBytes(cacheSize);
}
diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h
index 558f01f5e80..9a07c0ceab2 100644
--- a/searchlib/src/vespa/searchlib/docstore/visitcache.h
+++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h
@@ -56,7 +56,6 @@ public:
BlobSet &operator = (BlobSet &&) = default;
~BlobSet();
void append(uint32_t lid, vespalib::ConstBufferRef blob);
- void remove(uint32_t lid);
const Positions & getPositions() const { return _positions; }
vespalib::ConstBufferRef get(uint32_t lid) const;
vespalib::ConstBufferRef getBuffer() const { return vespalib::ConstBufferRef(_buffer.data(), _buffer.size()); }
@@ -75,7 +74,7 @@ class CompressedBlobSet {
public:
using CompressionConfig = vespalib::compression::CompressionConfig;
CompressedBlobSet();
- CompressedBlobSet(const CompressionConfig &compression, const BlobSet & uncompressed);
+ CompressedBlobSet(CompressionConfig compression, const BlobSet & uncompressed);
CompressedBlobSet(CompressedBlobSet && rhs) = default;
CompressedBlobSet & operator=(CompressedBlobSet && rhs) = default;
CompressedBlobSet(const CompressedBlobSet & rhs) = default;
@@ -98,14 +97,14 @@ private:
class VisitCache {
public:
using CompressionConfig = vespalib::compression::CompressionConfig;
- VisitCache(IDataStore &store, size_t cacheSize, const CompressionConfig &compression);
+ VisitCache(IDataStore &store, size_t cacheSize, CompressionConfig compression);
CompressedBlobSet read(const IDocumentStore::LidVector & keys) const;
void remove(uint32_t key);
void invalidate(uint32_t key) { remove(key); }
vespalib::CacheStats getCacheStats() const;
- void reconfigure(size_t cacheSize, const CompressionConfig &compression);
+ void reconfigure(size_t cacheSize, CompressionConfig compression);
private:
/**
* This implments the interface the cache uses when it has a cache miss.
@@ -115,18 +114,18 @@ private:
*/
class BackingStore {
public:
- BackingStore(IDataStore &store, const CompressionConfig &compression) :
+ BackingStore(IDataStore &store, CompressionConfig compression) :
_backingStore(store),
_compression(compression)
{ }
bool read(const KeySet &key, CompressedBlobSet &blobs) const;
void write(const KeySet &, const CompressedBlobSet &) { }
void erase(const KeySet &) { }
- void reconfigure(const CompressionConfig &compression);
+ void reconfigure(CompressionConfig compression);
private:
IDataStore &_backingStore;
- CompressionConfig _compression;
+ std::atomic<CompressionConfig> _compression;
};
using CacheParams = vespalib::CacheParam<
diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h
index 4a34b9c9e05..37bb925f11e 100644
--- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h
+++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h
@@ -26,12 +26,12 @@ public:
using CompressionConfig = vespalib::compression::CompressionConfig;
Config() : Config({CompressionConfig::LZ4, 9, 60}, 0x10000) { }
- Config(const CompressionConfig &compression, size_t maxChunkBytes)
+ Config(CompressionConfig compression, size_t maxChunkBytes)
: _compression(compression),
_maxChunkBytes(maxChunkBytes)
{ }
- const CompressionConfig & getCompression() const { return _compression; }
+ CompressionConfig getCompression() const { return _compression; }
size_t getMaxChunkBytes() const { return _maxChunkBytes; }
bool operator == (const Config & rhs) const {
return (_compression == rhs._compression) && (_maxChunkBytes == rhs._maxChunkBytes);
diff --git a/vespalib/src/tests/compression/compression_test.cpp b/vespalib/src/tests/compression/compression_test.cpp
index eb1fce0a683..27326243b60 100644
--- a/vespalib/src/tests/compression/compression_test.cpp
+++ b/vespalib/src/tests/compression/compression_test.cpp
@@ -67,6 +67,11 @@ TEST("requiret that zstd compression/decompression works") {
EXPECT_EQUAL(_G_compressableText, vespalib::string(decompress.data(), decompress.size()));
}
+TEST("require that CompressionConfig is Atomic") {
+ EXPECT_EQUAL(8u, sizeof(CompressionConfig));
+ EXPECT_TRUE(std::atomic<CompressionConfig>::is_always_lock_free);
+}
+
TEST_MAIN() {
TEST_RUN_ALL();
}
diff --git a/vespalib/src/vespa/vespalib/util/compressionconfig.h b/vespalib/src/vespa/vespalib/util/compressionconfig.h
index 58ea101db13..2a09fa7e086 100644
--- a/vespalib/src/vespa/vespalib/util/compressionconfig.h
+++ b/vespalib/src/vespa/vespalib/util/compressionconfig.h
@@ -9,7 +9,7 @@
namespace vespalib::compression {
struct CompressionConfig {
- enum Type {
+ enum Type : uint8_t {
NONE = 0,
NONE_MULTI = 1,
HISTORIC_2 = 2,
@@ -21,15 +21,15 @@ struct CompressionConfig {
};
CompressionConfig() noexcept
- : type(NONE), compressionLevel(0), threshold(90), minSize(0) {}
+ : CompressionConfig(NONE, 0, 90) {}
CompressionConfig(Type t) noexcept
- : type(t), compressionLevel(9), threshold(90), minSize(0) {}
+ : CompressionConfig(t, 9, 90) {}
CompressionConfig(Type t, uint8_t level, uint8_t minRes) noexcept
- : type(t), compressionLevel(level), threshold(minRes), minSize(0) {}
+ : CompressionConfig(t, level, minRes, 0) {}
CompressionConfig(Type t, uint8_t lvl, uint8_t minRes, size_t minSz) noexcept
- : type(t), compressionLevel(lvl), threshold(minRes), minSize(minSz) {}
+ : minSize(minSz), type(t), compressionLevel(lvl), threshold(minRes) {}
bool operator==(const CompressionConfig& o) const {
return (type == o.type
@@ -66,10 +66,10 @@ struct CompressionConfig {
}
bool useCompression() const { return isCompressed(type); }
- Type type;
- uint8_t compressionLevel;
- uint8_t threshold;
- size_t minSize;
+ uint32_t minSize;
+ Type type;
+ uint8_t compressionLevel;
+ uint8_t threshold;
};
}
diff --git a/vespalib/src/vespa/vespalib/util/compressor.cpp b/vespalib/src/vespa/vespalib/util/compressor.cpp
index c477b021956..4d708473423 100644
--- a/vespalib/src/vespa/vespalib/util/compressor.cpp
+++ b/vespalib/src/vespa/vespalib/util/compressor.cpp
@@ -15,7 +15,7 @@ namespace vespalib::compression {
namespace {
template <typename F>
-void with_compressor(const CompressionConfig::Type &type, F &&f) {
+void with_compressor(CompressionConfig::Type type, F &&f) {
switch (type) {
case CompressionConfig::LZ4:
{
@@ -40,7 +40,7 @@ void with_compressor(const CompressionConfig::Type &type, F &&f) {
//-----------------------------------------------------------------------------
CompressionConfig::Type
-compress(ICompressor & compressor, const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest)
+compress(ICompressor & compressor, CompressionConfig compression, const ConstBufferRef & org, DataBuffer & dest)
{
CompressionConfig::Type type(CompressionConfig::NONE);
dest.ensureFree(compressor.adjustProcessLen(0, org.size()));
@@ -55,7 +55,7 @@ compress(ICompressor & compressor, const CompressionConfig & compression, const
}
CompressionConfig::Type
-docompress(const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest)
+docompress(CompressionConfig compression, const ConstBufferRef & org, DataBuffer & dest)
{
switch (compression.type) {
case CompressionConfig::LZ4:
@@ -80,8 +80,9 @@ CompressionConfig::Type
compress(CompressionConfig::Type compression, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap) {
return compress(CompressionConfig(compression), org, dest, allowSwap);
}
+
CompressionConfig::Type
-compress(const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap)
+compress(CompressionConfig compression, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap)
{
CompressionConfig::Type type(CompressionConfig::NONE);
if (org.size() >= compression.minSize) {
@@ -124,7 +125,7 @@ decompress(ICompressor & decompressor, size_t uncompressedLen, const ConstBuffer
}
void
-decompress(const CompressionConfig::Type & type, size_t uncompressedLen, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap)
+decompress(CompressionConfig::Type type, size_t uncompressedLen, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap)
{
switch (type) {
case CompressionConfig::LZ4:
@@ -169,7 +170,7 @@ size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t payloadSize
//-----------------------------------------------------------------------------
-Compress::Compress(const CompressionConfig &config,
+Compress::Compress(CompressionConfig config,
const char *uncompressed_data, size_t uncompressed_size)
: _space(),
_type(CompressionConfig::NONE),
@@ -194,7 +195,7 @@ Compress::Compress(const CompressionConfig &config,
}
}
-Decompress::Decompress(const CompressionConfig::Type &type, size_t uncompressed_size,
+Decompress::Decompress(CompressionConfig::Type type, size_t uncompressed_size,
const char *compressed_data, size_t compressed_size)
: _space(),
_data(compressed_data),
diff --git a/vespalib/src/vespa/vespalib/util/compressor.h b/vespalib/src/vespa/vespalib/util/compressor.h
index fd29c028af2..2a02d684719 100644
--- a/vespalib/src/vespa/vespalib/util/compressor.h
+++ b/vespalib/src/vespa/vespalib/util/compressor.h
@@ -13,7 +13,7 @@ class ICompressor
{
public:
virtual ~ICompressor() { }
- virtual bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) = 0;
+ virtual bool process(CompressionConfig config, const void * input, size_t inputLen, void * output, size_t & outputLen) = 0;
virtual bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) = 0;
virtual size_t adjustProcessLen(uint16_t options, size_t len) const = 0;
};
@@ -28,7 +28,7 @@ public:
* @param allowSwap will tell it the data must be appended or if it can be swapped in if it is uncompressable or config is NONE.
*/
CompressionConfig::Type compress(CompressionConfig::Type compression, const ConstBufferRef & org, DataBuffer & dest, bool allowSwap);
-CompressionConfig::Type compress(const CompressionConfig & compression, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap);
+CompressionConfig::Type compress(CompressionConfig compression, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap);
/**
* Will try to decompress a buffer according to the config.
@@ -41,7 +41,7 @@ CompressionConfig::Type compress(const CompressionConfig & compression, const ve
* Then it will be swapped in.
* @param allowSwap will tell it the data must be appended or if it can be swapped in if compression type is NONE.
*/
-void decompress(const CompressionConfig::Type & compression, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap);
+void decompress(CompressionConfig::Type compression, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap);
size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t uncompressedSize);
@@ -53,14 +53,13 @@ size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t uncompresse
**/
class Compress {
private:
- alloc::Alloc _space;
+ alloc::Alloc _space;
CompressionConfig::Type _type;
- const char *_data;
- size_t _size;
+ const char *_data;
+ size_t _size;
public:
- Compress(const CompressionConfig &config,
- const char *uncompressed_data, size_t uncompressed_size);
- const CompressionConfig::Type &type() const { return _type; }
+ Compress(CompressionConfig config, const char *uncompressed_data, size_t uncompressed_size);
+ CompressionConfig::Type type() const { return _type; }
const char *data() const { return _data; }
size_t size() const { return _size; }
};
@@ -72,10 +71,10 @@ public:
class Decompress {
private:
alloc::Alloc _space;
- const char *_data;
- size_t _size;
+ const char *_data;
+ size_t _size;
public:
- Decompress(const CompressionConfig::Type &type, size_t uncompressed_size,
+ Decompress(CompressionConfig::Type type, size_t uncompressed_size,
const char *compressed_data, size_t compressed_size);
const char *data() const { return _data; }
size_t size() const { return _size; }
diff --git a/vespalib/src/vespa/vespalib/util/lz4compressor.cpp b/vespalib/src/vespa/vespalib/util/lz4compressor.cpp
index d33b7339c4b..4176dd5d2c6 100644
--- a/vespalib/src/vespa/vespalib/util/lz4compressor.cpp
+++ b/vespalib/src/vespa/vespalib/util/lz4compressor.cpp
@@ -13,7 +13,7 @@ namespace vespalib::compression {
size_t LZ4Compressor::adjustProcessLen(uint16_t, size_t len) const { return LZ4_compressBound(len); }
bool
-LZ4Compressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV)
+LZ4Compressor::process(CompressionConfig config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV)
{
const char * input(static_cast<const char *>(inputV));
char * output(static_cast<char *>(outputV));
diff --git a/vespalib/src/vespa/vespalib/util/lz4compressor.h b/vespalib/src/vespa/vespalib/util/lz4compressor.h
index e723fce80cf..947888e625f 100644
--- a/vespalib/src/vespa/vespalib/util/lz4compressor.h
+++ b/vespalib/src/vespa/vespalib/util/lz4compressor.h
@@ -8,7 +8,7 @@ namespace vespalib::compression {
class LZ4Compressor : public ICompressor
{
public:
- bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) override;
+ bool process(CompressionConfig config, const void * input, size_t inputLen, void * output, size_t & outputLen) override;
bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) override;
size_t adjustProcessLen(uint16_t options, size_t len) const override;
};
diff --git a/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp b/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp
index 1b0562985d8..51adc68c848 100644
--- a/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp
+++ b/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp
@@ -36,7 +36,7 @@ thread_local std::unique_ptr<DecompressContext> _tlDecompressState;
size_t ZStdCompressor::adjustProcessLen(uint16_t, size_t len) const { return ZSTD_compressBound(len); }
bool
-ZStdCompressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV)
+ZStdCompressor::process(CompressionConfig config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV)
{
size_t maxOutputLen = ZSTD_compressBound(inputLen);
if ( ! _tlCompressState) {
diff --git a/vespalib/src/vespa/vespalib/util/zstdcompressor.h b/vespalib/src/vespa/vespalib/util/zstdcompressor.h
index 1a141ab2b0f..483f2521475 100644
--- a/vespalib/src/vespa/vespalib/util/zstdcompressor.h
+++ b/vespalib/src/vespa/vespalib/util/zstdcompressor.h
@@ -8,7 +8,7 @@ namespace vespalib::compression {
class ZStdCompressor : public ICompressor
{
public:
- bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) override;
+ bool process(CompressionConfig config, const void * input, size_t inputLen, void * output, size_t & outputLen) override;
bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) override;
size_t adjustProcessLen(uint16_t options, size_t len) const override;
};