diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2017-10-05 23:06:08 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2017-10-05 23:07:08 +0200 |
commit | 71d8477b68edd3dede0734c135914295f761d2e9 (patch) | |
tree | 785e01e4e9a9b6d79ab8d38af1e75ad4bcfeec0b | |
parent | fee8f7f7aef4602da89d5bf43620a064d0de7172 (diff) |
Add equality operators on the configs.
12 files changed, 184 insertions, 222 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 71302e858f2..166b270c269 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -48,21 +48,16 @@ class ShrinkSummaryLidSpaceFlushTarget : public ShrinkLidSpaceFlushTarget searchcorespi::index::IThreadService & _summaryService; public: - ShrinkSummaryLidSpaceFlushTarget(const vespalib::string &name, - Type type, - Component component, - SerialNum flushedSerialNum, - Time lastFlushTime, + ShrinkSummaryLidSpaceFlushTarget(const vespalib::string &name, Type type, Component component, + SerialNum flushedSerialNum, Time lastFlushTime, searchcorespi::index::IThreadService & summaryService, std::shared_ptr<ICompactableLidSpace> target); ~ShrinkSummaryLidSpaceFlushTarget(); virtual Task::UP initFlush(SerialNum currentSerial) override; }; -ShrinkSummaryLidSpaceFlushTarget::ShrinkSummaryLidSpaceFlushTarget(const vespalib::string &name, - Type type, - Component component, - SerialNum flushedSerialNum, +ShrinkSummaryLidSpaceFlushTarget::ShrinkSummaryLidSpaceFlushTarget(const vespalib::string &name, Type type, + Component component, SerialNum flushedSerialNum, Time lastFlushTime, searchcorespi::index::IThreadService & summaryService, std::shared_ptr<ICompactableLidSpace> target) @@ -85,13 +80,9 @@ ShrinkSummaryLidSpaceFlushTarget::initFlush(SerialNum currentSerial) } SummaryManager::SummarySetup:: -SummarySetup(const vespalib::string & baseDir, - const DocTypeName & docTypeName, - const SummaryConfig & summaryCfg, - const SummarymapConfig & summarymapCfg, - const JuniperrcConfig & juniperCfg, - const search::IAttributeManager::SP &attributeMgr, - const search::IDocumentStore::SP & docStore, +SummarySetup(const vespalib::string & baseDir, const DocTypeName & docTypeName, const SummaryConfig & summaryCfg, + const SummarymapConfig & summarymapCfg, const JuniperrcConfig & juniperCfg, + const search::IAttributeManager::SP &attributeMgr, const search::IDocumentStore::SP & docStore, const DocumentTypeRepo::SP &repo) : _docsumWriter(), _wordFolder(), @@ -186,23 +177,21 @@ deriveConfig(const ProtonConfig::Summary & summary) { const ProtonConfig::Summary::Log::Chunk & chunk(log.chunk); WriteableFileChunk::Config fileConfig(deriveCompression(chunk.compression), chunk.maxbytes); - LogDataStore::Config logConfig(log.maxfilesize, log.maxdiskbloatfactor, log.maxbucketspread, - log.minfilesizefactor, log.numthreads, log.compact2activefile, - deriveCompression(log.compact.compression), fileConfig); - logConfig.disableCrcOnRead(chunk.skipcrconread); + LogDataStore::Config logConfig; + logConfig.setMaxFileSize(log.maxfilesize).setMaxDiskBloatFactor(log.maxdiskbloatfactor) + .setMaxBucketSpread(log.maxbucketspread).setMinFileSizeFactor(log.minfilesizefactor) + .setNumThreads(log.numthreads).compact2ActiveFile(log.compact2activefile) + .compactCompression(deriveCompression(log.compact.compression)).setFileConfig(fileConfig) + .disableCrcOnRead(chunk.skipcrconread); return LogDocumentStore::Config(config, logConfig); } } -SummaryManager::SummaryManager(vespalib::ThreadExecutor & executor, - const ProtonConfig::Summary & summary, - const search::GrowStrategy & growStrategy, - const vespalib::string &baseDir, - const DocTypeName &docTypeName, - const TuneFileSummary &tuneFileSummary, - const FileHeaderContext &fileHeaderContext, - search::transactionlog::SyncProxy &tlSyncer, +SummaryManager::SummaryManager(vespalib::ThreadExecutor & executor, const ProtonConfig::Summary & summary, + const search::GrowStrategy & growStrategy, const vespalib::string &baseDir, + const DocTypeName &docTypeName, const TuneFileSummary &tuneFileSummary, + const FileHeaderContext &fileHeaderContext, search::transactionlog::SyncProxy &tlSyncer, const search::IBucketizer::SP & bucketizer) : _baseDir(baseDir), _docTypeName(docTypeName), @@ -210,8 +199,7 @@ SummaryManager::SummaryManager(vespalib::ThreadExecutor & executor, _tuneFileSummary(tuneFileSummary), _currentSerial(0u) { - _docStore.reset(new LogDocumentStore(executor, baseDir, - deriveConfig(summary), + _docStore.reset(new LogDocumentStore(executor, baseDir, deriveConfig(summary), growStrategy, tuneFileSummary, fileHeaderContext, tlSyncer, summary.compact2buckets ? bucketizer : search::IBucketizer::SP())); } diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index e8c2173a87f..b48d754bfb5 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/searchlib/docstore/documentstore.h> +#include <vespa/searchlib/docstore/logdocumentstore.h> #include <vespa/searchlib/docstore/cachestats.h> #include <vespa/document/repo/documenttyperepo.h> @@ -28,19 +28,19 @@ struct NullDataStore : IDataStore { fastos::TimeStamp getLastFlushTime() const override { return fastos::TimeStamp(); } void accept(IDataStoreVisitor &, IDataStoreVisitorProgress &, bool) override { } double getVisitCost() const override { return 1.0; } - virtual DataStoreStorageStats getStorageStats() const override { + DataStoreStorageStats getStorageStats() const override { return DataStoreStorageStats(0, 0, 0.0, 0, 0, 0); } - virtual MemoryUsage getMemoryUsage() const override { return MemoryUsage(); } - virtual std::vector<DataStoreFileChunkStats> + MemoryUsage getMemoryUsage() const override { return MemoryUsage(); } + std::vector<DataStoreFileChunkStats> getFileChunkStats() const override { std::vector<DataStoreFileChunkStats> result; return result; } - virtual void compactLidSpace(uint32_t wantedDocLidLimit) override { (void) wantedDocLidLimit; } - virtual bool canShrinkLidSpace() const override { return false; } - virtual size_t getEstimatedShrinkLidSpaceGain() const override { return 0; } - virtual void shrinkLidSpace() override {} + void compactLidSpace(uint32_t wantedDocLidLimit) override { (void) wantedDocLidLimit; } + bool canShrinkLidSpace() const override { return false; } + size_t getEstimatedShrinkLidSpaceGain() const override { return 0; } + void shrinkLidSpace() override {} }; TEST_FFF("require that uncache docstore lookups are counted", @@ -61,4 +61,23 @@ TEST_FFF("require that cached docstore lookups are counted", EXPECT_EQUAL(1u, f3.getCacheStats().misses); } +TEST("require that DocumentStore::Config equality operator detects inequality") { + using C = DocumentStore::Config; + EXPECT_TRUE(C() == C()); + EXPECT_TRUE(C(CompressionConfig::NONE, 100000, 100) == C(CompressionConfig::NONE, 100000, 100)); + EXPECT_FALSE(C(CompressionConfig::NONE, 100000, 100) == C(CompressionConfig::NONE, 100000, 99)); + EXPECT_FALSE(C(CompressionConfig::NONE, 100000, 100) == C(CompressionConfig::NONE, 100001, 100)); + EXPECT_FALSE(C(CompressionConfig::NONE, 100000, 100) == C(CompressionConfig::LZ4, 100000, 100)); +} + +TEST("require that LogDocumentStore::Config equality operator detects inequality") { + using C = LogDocumentStore::Config; + using LC = LogDataStore::Config; + using DC = DocumentStore::Config; + EXPECT_TRUE(C() == C()); + EXPECT_FALSE(C() != C()); + EXPECT_FALSE(C(DC(CompressionConfig::NONE, 100000, 100), LC()) == C()); + EXPECT_FALSE(C(DC(), LC().setNumThreads(7)) == C()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index 1c7053500c7..7e4b4fa4363 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -28,8 +28,7 @@ using vespalib::compression::CompressionConfig; using vespalib::asciistream; using index::DummyFileHeaderContext; -namespace -{ +namespace { const string doc_type_name = "test"; const string header_name = doc_type_name + ".header"; @@ -80,16 +79,8 @@ class MyTlSyncer : public transactionlog::SyncProxy SerialNum _syncedTo; public: - MyTlSyncer() - : _syncedTo(0) - { - } - - void - sync(SerialNum syncTo) override - { - _syncedTo = syncTo; - } + MyTlSyncer() : _syncedTo(0) {} + void sync(SerialNum syncTo) override { _syncedTo = syncTo; } }; @@ -106,9 +97,7 @@ public: MyVisitorBase(DocumentTypeRepo &repo, uint32_t docIdLimit, bool before); }; -MyVisitorBase::MyVisitorBase(DocumentTypeRepo &repo, - uint32_t docIdLimit, - bool before) +MyVisitorBase::MyVisitorBase(DocumentTypeRepo &repo, uint32_t docIdLimit, bool before) : _repo(repo), _visitCount(0u), _visitRmCount(0u), @@ -125,11 +114,8 @@ class MyVisitor : public MyVisitorBase, public: using MyVisitorBase::MyVisitorBase; - virtual void - visit(uint32_t lid, const Document &doc) override; - - virtual void - visit(uint32_t lid) override; + void visit(uint32_t lid, const Document &doc) override; + void visit(uint32_t lid) override; }; @@ -184,11 +170,8 @@ public: MyVisitorProgress(); - virtual void - updateProgress(double progress) override; - - virtual double - getProgress() const; + void updateProgress(double progress) override; + double getProgress() const; }; @@ -232,52 +215,27 @@ struct Fixture BitVector::UP _valid; Fixture(); - ~Fixture(); - Document::UP - makeDoc(uint32_t i); - - void - resetDocStore(); - - void - mkdir(); - - void - rmdir(); - - void - setDocIdLimit(uint32_t docIdLimit); - - void - put(const Document &doc, uint32_t lid); - - void - remove(uint32_t lid); - - void - flush(); - - void - populate(uint32_t low, uint32_t high, uint32_t docIdLimit); - - void - applyRemoves(uint32_t rmDocs); - - void - checkRemovePostCond(uint32_t numDocs, - uint32_t docIdLimit, - uint32_t rmDocs, - bool before); + Document::UP makeDoc(uint32_t i); + void resetDocStore(); + void mkdir(); + void rmdir(); + void setDocIdLimit(uint32_t docIdLimit); + void put(const Document &doc, uint32_t lid); + void remove(uint32_t lid); + void flush(); + void populate(uint32_t low, uint32_t high, uint32_t docIdLimit); + void applyRemoves(uint32_t rmDocs); + void checkRemovePostCond(uint32_t numDocs, uint32_t docIdLimit, uint32_t rmDocs, bool before); }; Fixture::Fixture() : _baseDir("visitor"), _repo(makeDocTypeRepoConfig()), _storeConfig(DocumentStore::Config(CompressionConfig::NONE, 0, 0), - LogDataStore::Config(50000, 0.2, 3.0, 0.2, 1, true, CompressionConfig::LZ4, - WriteableFileChunk::Config(CompressionConfig(), 16384))), + LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0).setNumThreads(1) + .setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16384))), _executor(_storeConfig.getLogConfig().getNumThreads(), 128 * 1024), _fileHeaderContext(), _tlSyncer(), @@ -307,14 +265,8 @@ Fixture::makeDoc(uint32_t i) void Fixture::resetDocStore() { - _store.reset(new LogDocumentStore(_executor, - _baseDir, - _storeConfig, - GrowStrategy(), - TuneFileSummary(), - _fileHeaderContext, - _tlSyncer, - NULL)); + _store.reset(new LogDocumentStore(_executor, _baseDir, _storeConfig, GrowStrategy(), + TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr)); } diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index 598913a3222..31c21723cd0 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -11,6 +11,8 @@ #include <iostream> #include <vespa/log/log.h> +#include <vespa/vespalib/util/compressionconfig.h> + LOG_SETUP("file_chunk_test"); using namespace search; @@ -210,5 +212,16 @@ TEST("require that entries with lid >= docIdLimit are skipped in updateLidMap()" } } +using vespalib::compression::CompressionConfig; + +TEST("require that operator == detects inequality") { + using C = WriteableFileChunk::Config; + EXPECT_TRUE(C() == C()); + EXPECT_TRUE(C({}, 1) == C({}, 1)); + EXPECT_FALSE(C({}, 2) == C({}, 1)); + EXPECT_FALSE(C({}, 1) == C({}, 2)); + EXPECT_FALSE(C({CompressionConfig::LZ4, 9, 60}, 2) == C({}, 2)); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index ed6afb06681..c7da0c17c80 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -202,20 +202,17 @@ TEST("test that DirectIOPadding works accordng to spec") { TEST("testGrowing") { FastOS_File::EmptyAndRemoveDirectory("growing"); EXPECT_TRUE(FastOS_File::MakeDirectory("growing")); - LogDataStore::Config config(100000, 0.1, 3.0, 0.2, 8, true, CompressionConfig::LZ4, - WriteableFileChunk::Config(CompressionConfig(CompressionConfig::LZ4, 9, 60), 1000)); + LogDataStore::Config config; //(100000, 0.1, 3.0, 0.2, 8, true, CompressionConfig::LZ4, + // WriteableFileChunk::Config(CompressionConfig(CompressionConfig::LZ4, 9, 60), 1000)); + config.setMaxFileSize(100000).setMaxDiskBloatFactor(0.1).setMaxBucketSpread(3.0).setMinFileSizeFactor(0.2) + .setNumThreads(8).compact2ActiveFile(true).compactCompression({CompressionConfig::LZ4}) + .setFileConfig({{CompressionConfig::LZ4, 9, 60}, 1000}); vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); DummyFileHeaderContext fileHeaderContext; MyTlSyncer tlSyncer; { - LogDataStore datastore(executor, - "growing", - config, - GrowStrategy(), - TuneFileSummary(), - fileHeaderContext, - tlSyncer, - NULL); + LogDataStore datastore(executor, "growing", config, GrowStrategy(), + TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); srand(7); char buffer[12000]; SerialNum lastSyncToken(0); @@ -245,14 +242,8 @@ TEST("testGrowing") { checkStats(datastore, 31000, 30000); } { - LogDataStore datastore(executor, - "growing", - config, - GrowStrategy(), - TuneFileSummary(), - fileHeaderContext, - tlSyncer, - NULL); + LogDataStore datastore(executor, "growing", config, GrowStrategy(), + TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); checkStats(datastore, 30000, 30000); } @@ -527,14 +518,13 @@ VisitCacheStore::VisitCacheStore() : _myDir("visitcache"), _repo(makeDocTypeRepoConfig()), _config(DocumentStore::Config(CompressionConfig::LZ4, 1000000, 0).allowVisitCaching(true), - LogDataStore::Config(50000, 0.2, 3.0, 0.2, 1, true,CompressionConfig::LZ4, - WriteableFileChunk::Config(CompressionConfig(), 16384))), + LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0).setNumThreads(1) + .setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16384))), _fileHeaderContext(), _executor(_config.getLogConfig().getNumThreads(), 128*1024), _tlSyncer(), - _datastore(_executor, _myDir.getDir(), _config, - GrowStrategy(), TuneFileSummary(), - _fileHeaderContext, _tlSyncer, NULL), + _datastore(_executor, _myDir.getDir(), _config, GrowStrategy(), + TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr), _inserted(), _serial(1) { } @@ -696,14 +686,8 @@ TEST("requireThatFlushTimeIsAvailableAfterFlush") { LogDataStore::Config config; vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); MyTlSyncer tlSyncer; - LogDataStore store(executor, - testDir.getDir(), - config, - GrowStrategy(), - TuneFileSummary(), - fileHeaderContext, - tlSyncer, - NULL); + LogDataStore store(executor, testDir.getDir(), config, GrowStrategy(), + TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); EXPECT_EQUAL(0, store.getLastFlushTime().time()); uint64_t flushToken = store.initFlush(5); EXPECT_EQUAL(5u, flushToken); @@ -757,9 +741,7 @@ TEST("testBucketDensityComputer") { LogDataStore::Config getBasicConfig(size_t maxFileSize) { - CompressionConfig compCfg; - WriteableFileChunk::Config fileCfg; - return LogDataStore::Config(maxFileSize, 0.2, 2.5, 0.2, 1, true, compCfg, fileCfg); + return LogDataStore::Config().setMaxFileSize(maxFileSize); } vespalib::string @@ -794,14 +776,8 @@ struct Fixture { serialNum(0), fileHeaderCtx(), tlSyncer(), - store(executor, - dirName, - getBasicConfig(maxFileSize), - GrowStrategy(), - TuneFileSummary(), - fileHeaderCtx, - tlSyncer, - nullptr) + store(executor, dirName, getBasicConfig(maxFileSize), GrowStrategy(), + TuneFileSummary(), fileHeaderCtx, tlSyncer, nullptr) { dir.cleanup(dirCleanup); } @@ -997,6 +973,20 @@ TEST("require that findIncompleteCompactedFiles does expected filtering") { } +TEST("require that config equality operator detects inequality") { + using C = LogDataStore::Config; + EXPECT_TRUE(C() == C()); + EXPECT_FALSE(C() == C().setMaxFileSize(1)); + EXPECT_FALSE(C() == C().setMaxDiskBloatFactor(0.3)); + EXPECT_FALSE(C() == C().setMaxBucketSpread(0.3)); + EXPECT_FALSE(C() == C().setMinFileSizeFactor(0.3)); + EXPECT_FALSE(C() == C().setNumThreads(3)); + EXPECT_FALSE(C() == C().setFileConfig(WriteableFileChunk::Config({}, 70))); + EXPECT_FALSE(C() == C().disableCrcOnRead(true)); + EXPECT_FALSE(C() == C().compact2ActiveFile(false)); + EXPECT_FALSE(C() == C().compactCompression({CompressionConfig::ZSTD})); +} + TEST_MAIN() { DummyFileHeaderContext::setCreator("logdatastore_test"); TEST_RUN_ALL(); diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp index 63b924d5cc2..42920a093eb 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp @@ -196,6 +196,15 @@ public: using VisitCache = docstore::VisitCache; using docstore::Value; +bool +DocumentStore::Config::operator == (const Config &rhs) const { + return (_maxCacheBytes == rhs._maxCacheBytes) && + (_allowVisitCaching == rhs._allowVisitCaching) && + (_initialCacheEntries == rhs._initialCacheEntries) && + (_compression == rhs._compression); +} + + DocumentStore::DocumentStore(const Config & config, IDataStore & store) : IDocumentStore(), _config(config), diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h index af173862295..e6e4364e6ab 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h @@ -44,6 +44,7 @@ public: size_t getInitialCacheEntries() const { return _initialCacheEntries; } bool allowVisitCaching() const { return _allowVisitCaching; } Config & allowVisitCaching(bool allow) { _allowVisitCaching = allow; return *this; } + bool operator == (const Config &) const; private: CompressionConfig _compression; size_t _maxCacheBytes; diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 53455cb1223..6c2509c53c5 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -29,15 +29,35 @@ using docstore::StoreByBucket; using docstore::BucketCompacter; using namespace std::literals; -LogDataStore::LogDataStore(vespalib::ThreadExecutor &executor, - const vespalib::string &dirName, - const Config &config, - const GrowStrategy &growStrategy, - const TuneFileSummary &tune, - const FileHeaderContext &fileHeaderContext, - transactionlog::SyncProxy &tlSyncer, - const IBucketizer::SP & bucketizer, - bool readOnly) +LogDataStore::Config::Config() + : _maxFileSize(1000000000ul), + _maxDiskBloatFactor(0.2), + _maxBucketSpread(2.5), + _minFileSizeFactor(0.2), + _numThreads(8), + _skipCrcOnRead(false), + _compact2ActiveFile(true), + _compactCompression(CompressionConfig::LZ4), + _fileConfig() +{ } + +bool +LogDataStore::Config::operator == (const Config & rhs) const { + return (_maxBucketSpread == rhs._maxBucketSpread) && + (_maxDiskBloatFactor == rhs._maxDiskBloatFactor) && + (_maxFileSize == rhs._maxFileSize) && + (_minFileSizeFactor == rhs._minFileSizeFactor) && + (_compact2ActiveFile == rhs._compact2ActiveFile) && + (_skipCrcOnRead == rhs._skipCrcOnRead) && + (_numThreads == rhs._numThreads) && + (_compactCompression == rhs._compactCompression) && + (_fileConfig == rhs._fileConfig); +} + +LogDataStore::LogDataStore(vespalib::ThreadExecutor &executor, const vespalib::string &dirName, const Config &config, + const GrowStrategy &growStrategy, const TuneFileSummary &tune, + const FileHeaderContext &fileHeaderContext, transactionlog::SyncProxy &tlSyncer, + const IBucketizer::SP & bucketizer, bool readOnly) : IDataStore(dirName), _config(config), _tune(tune), diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index ab1c0026560..5928602e196 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -36,36 +36,16 @@ public: using CompressionConfig = vespalib::compression::CompressionConfig; class Config { public: - Config() - : _maxFileSize(1000000000ul), - _maxDiskBloatFactor(0.2), - _maxBucketSpread(2.5), - _minFileSizeFactor(0.2), - _numThreads(8), - _skipCrcOnRead(false), - _compactToActiveFile(true), - _compactCompression(CompressionConfig::LZ4), - _fileConfig() - { } - - Config(size_t maxFileSize, - double maxDiskBloatFactor, - double maxBucketSpread, - double minFileSizeFactor, - size_t numThreads, - bool compactToActiveFile, - const CompressionConfig & compactCompression, - const WriteableFileChunk::Config & fileConfig) - : _maxFileSize(maxFileSize), - _maxDiskBloatFactor(maxDiskBloatFactor), - _maxBucketSpread(maxBucketSpread), - _minFileSizeFactor(minFileSizeFactor), - _numThreads(numThreads), - _skipCrcOnRead(false), - _compactToActiveFile(compactToActiveFile), - _compactCompression(compactCompression), - _fileConfig(fileConfig) - { } + Config(); + + Config & setMaxFileSize(size_t v) { _maxFileSize = v; return *this; } + Config & setMaxDiskBloatFactor(double v) { _maxDiskBloatFactor = v; return *this; } + Config & setMaxBucketSpread(double v) { _maxBucketSpread = v; return *this; } + Config & setMinFileSizeFactor(double v) { _minFileSizeFactor = v; return *this; } + + Config & setNumThreads(size_t v) { _numThreads = v; return *this; } + Config & compactCompression(CompressionConfig v) { _compactCompression = v; return *this; } + Config & setFileConfig(WriteableFileChunk::Config v) { _fileConfig = v; return *this; } size_t getMaxFileSize() const { return _maxFileSize; } double getMaxDiskBloatFactor() const { return _maxDiskBloatFactor; } @@ -74,10 +54,14 @@ public: size_t getNumThreads() const { return _numThreads; } bool crcOnReadDisabled() const { return _skipCrcOnRead; } - bool compact2ActiveFile() const { return _compactToActiveFile; } + bool compact2ActiveFile() const { return _compact2ActiveFile; } const CompressionConfig & compactCompression() const { return _compactCompression; } + const WriteableFileChunk::Config & getFileConfig() const { return _fileConfig; } Config & disableCrcOnRead(bool v) { _skipCrcOnRead = v; return *this;} + Config & compact2ActiveFile(bool v) { _compact2ActiveFile = v; return *this; } + + bool operator == (const Config &) const; private: size_t _maxFileSize; double _maxDiskBloatFactor; @@ -85,7 +69,7 @@ public: double _minFileSizeFactor; size_t _numThreads; bool _skipCrcOnRead; - bool _compactToActiveFile; + bool _compact2ActiveFile; CompressionConfig _compactCompression; WriteableFileChunk::Config _fileConfig; }; @@ -105,15 +89,10 @@ public: * The caller must keep it alive for the semantic * lifetime of the log data store. */ - LogDataStore(vespalib::ThreadExecutor &executor, - const vespalib::string &dirName, - const Config & config, - const GrowStrategy &growStrategy, - const TuneFileSummary &tune, + LogDataStore(vespalib::ThreadExecutor &executor, const vespalib::string &dirName, const Config & config, + const GrowStrategy &growStrategy, const TuneFileSummary &tune, const search::common::FileHeaderContext &fileHeaderContext, - transactionlog::SyncProxy &tlSyncer, - const IBucketizer::SP & bucketizer, - bool readOnly = false); + transactionlog::SyncProxy &tlSyncer, const IBucketizer::SP & bucketizer, bool readOnly = false); ~LogDataStore(); diff --git a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp index 04c4d40dd2c..e2b29f6bdd6 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp @@ -10,7 +10,7 @@ using common::FileHeaderContext; bool LogDocumentStore::Config::operator == (const Config & rhs) const { (void) rhs; - return true; + return DocumentStore::Config::operator ==(rhs) && (_logConfig == rhs._logConfig); } LogDocumentStore::LogDocumentStore(vespalib::ThreadExecutor & executor, diff --git a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h index 7b44a864ea0..3c9aefc9934 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.h @@ -44,14 +44,10 @@ public: * The caller must keep it alive for the semantic * lifetime of the log data store. */ - LogDocumentStore(vespalib::ThreadExecutor & executor, - const vespalib::string & baseDir, - const Config & config, - const GrowStrategy & growStrategy, - const TuneFileSummary &tuneFileSummary, + LogDocumentStore(vespalib::ThreadExecutor & executor, const vespalib::string & baseDir, const Config & config, + const GrowStrategy & growStrategy, const TuneFileSummary &tuneFileSummary, const common::FileHeaderContext &fileHeaderContext, - transactionlog::SyncProxy &tlSyncer, - const IBucketizer::SP & bucketizer); + transactionlog::SyncProxy &tlSyncer, const IBucketizer::SP & bucketizer); ~LogDocumentStore(); void reconfigure(const Config & config); private: diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index 2b21f12a314..f5923134138 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -23,10 +23,7 @@ public: { public: using CompressionConfig = vespalib::compression::CompressionConfig; - Config() - : _compression(CompressionConfig::LZ4, 9, 60), - _maxChunkBytes(0x10000) - { } + Config() : Config({CompressionConfig::LZ4, 9, 60}, 0x10000) { } Config(const CompressionConfig &compression, size_t maxChunkBytes) : _compression(compression), @@ -35,6 +32,9 @@ public: const CompressionConfig & getCompression() const { return _compression; } size_t getMaxChunkBytes() const { return _maxChunkBytes; } + bool operator == (const Config & rhs) const { + return (_compression == rhs._compression) && (_maxChunkBytes == rhs._maxChunkBytes); + } private: CompressionConfig _compression; size_t _maxChunkBytes; @@ -42,16 +42,11 @@ public: public: typedef std::unique_ptr<WriteableFileChunk> UP; - WriteableFileChunk(vespalib::ThreadExecutor & executor, - FileId fileId, NameId nameId, - const vespalib::string & baseName, - uint64_t initialSerialNum, - uint32_t docIdLimit, - const Config & config, - const TuneFileSummary &tune, - const common::FileHeaderContext &fileHeaderContext, - const IBucketizer * bucketizer, - bool crcOnReadDisabled); + WriteableFileChunk(vespalib::ThreadExecutor & executor, FileId fileId, NameId nameId, + const vespalib::string & baseName, uint64_t initialSerialNum, + uint32_t docIdLimit, const Config & config, + const TuneFileSummary &tune, const common::FileHeaderContext &fileHeaderContext, + const IBucketizer * bucketizer, bool crcOnReadDisabled); ~WriteableFileChunk(); ssize_t read(uint32_t lid, SubChunkId chunk, vespalib::DataBuffer & buffer) const override; |