From cb60fccb0fb3bf92bf2a223324dad44e986ae54d Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 6 Oct 2017 14:05:38 +0200 Subject: Let config reflect what it is, a threadpool serving a range of background task in the backend. --- searchcore/src/vespa/searchcore/config/proton.def | 7 +++ .../proton/server/documentdbconfigmanager.cpp | 5 +- .../src/vespa/searchcore/proton/server/proton.cpp | 70 ++++++++++------------ searchlib/src/apps/docstore/benchmarkdatastore.cpp | 2 +- .../src/apps/docstore/documentstoreinspect.cpp | 2 +- searchlib/src/apps/docstore/verifylogdatastore.cpp | 2 +- searchlib/src/apps/tests/biglogtest.cpp | 2 +- .../document_store/document_store_test.cpp | 2 +- .../document_store_visitor_test.cpp | 4 +- .../docstore/logdatastore/logdatastore_test.cpp | 49 +++++++-------- .../src/vespa/searchlib/docstore/logdatastore.cpp | 2 - .../src/vespa/searchlib/docstore/logdatastore.h | 3 - 12 files changed, 70 insertions(+), 80 deletions(-) diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 594a8717d7f..8c01c45dac8 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -256,6 +256,8 @@ summary.log.maxbucketspread double default=2.5 summary.log.minfilesizefactor double default=0.2 ## Number of threads used for compressing incomming documents/compacting. +## Deprecated. Use background.threads instead. +## TODO Remove summary.log.numthreads int default=8 restart ## Control io options during flush of stored documents. @@ -376,6 +378,11 @@ visit.ignoremaxbytes bool default=true ## When set to 0 (default) we use 1 separate thread per document database. initialize.threads int default = 0 +## Number of worker threads doing background compaction/compression tasks. +## They all live i a shared thread pool. +## When set to 0 (default), it will have enough threads to saturate half of the cores. +background.threads int default=0 + ## Portion of enumstore address space that can be used before put and update ## portion of feed is blocked. writefilter.attribute.enumstorelimit double default = 0.9 diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index f45fc658129..6aea5234fbe 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -170,9 +170,8 @@ deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::M logConfig.setMaxFileSize(log.maxfilesize) .setMaxDiskBloatFactor(std::min(flush.diskbloatfactor, flush.each.diskbloatfactor)) .setMaxBucketSpread(log.maxbucketspread).setMinFileSizeFactor(log.minfilesizefactor) - .setNumThreads(log.numthreads).compact2ActiveFile(log.compact2activefile) - .compactCompression(deriveCompression(log.compact.compression)).setFileConfig(fileConfig) - .disableCrcOnRead(chunk.skipcrconread); + .compact2ActiveFile(log.compact2activefile).compactCompression(deriveCompression(log.compact.compression)) + .setFileConfig(fileConfig).disableCrcOnRead(chunk.skipcrconread); return LogDocumentStore::Config(config, logConfig); } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 34cfaaeddb2..fbac7cb4278 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -85,12 +85,24 @@ diskMemUsageSamplerConfig(const ProtonConfig &proton, const HwInfo &hwInfo) hwInfo); } +size_t +deriveBackgroundThreads(const ProtonConfig & proton) { + size_t threads = std::thread::hardware_concurrency(); + if (proton.background.threads != 0) { + threads = proton.background.threads; + } + threads = std::max(threads, size_t(proton.summary.log.numthreads)); + + // We need at least 1 guaranteed free worker in order to ensure progress so #documentsdbs + 1 should suffice, + // but we will not be cheap and give #documentsdbs * 2 + return std::max(threads, proton.documentdb.size() * 2);; } -static const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component"; +const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component"; -Proton::ProtonFileHeaderContext::ProtonFileHeaderContext(const Proton &proton_, - const vespalib::string &creator) +} + +Proton::ProtonFileHeaderContext::ProtonFileHeaderContext(const Proton &proton_, const vespalib::string &creator) : _proton(proton_), _hostName(), _creator(creator), @@ -208,7 +220,7 @@ Proton::init() { assert( ! _initStarted && ! _initComplete ); _initStarted = true; - if (_threadPool.NewThread(&_clock, NULL) == NULL) { + if (_threadPool.NewThread(&_clock, nullptr) == nullptr) { throw IllegalStateException("Failed starting thread for the cheap clock"); } _protonConfigFetcher.start(); @@ -234,8 +246,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size); - _hwInfoSampler = std::make_unique(protonConfig.basedir, - samplerCfg); + _hwInfoSampler = std::make_unique(protonConfig.basedir, samplerCfg); _hwInfo = _hwInfoSampler->hwInfo(); setFS4Compression(protonConfig); _diskMemUsageSampler = std::make_unique @@ -244,8 +255,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _metricsEngine.reset(new MetricsEngine()); _metricsEngine->addMetricsHook(_metricsHook); - _fileHeaderContext.setClusterName(protonConfig.clustername, - protonConfig.basedir); + _fileHeaderContext.setClusterName(protonConfig.clustername, protonConfig.basedir); _tls.reset(new TLS(_configUri.createWithNewId(protonConfig.tlsconfigid), _fileHeaderContext)); _matchEngine.reset(new MatchEngine(protonConfig.numsearcherthreads, protonConfig.numthreadspersearch, @@ -257,7 +267,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) const ProtonConfig::Flush & flush(protonConfig.flush); switch (flush.strategy) { case ProtonConfig::Flush::MEMORY: { - MemoryFlush::SP memoryFlush = std::make_shared( + auto memoryFlush = std::make_shared( MemoryFlushConfigUpdater::convertConfig(flush.memory), fastos::ClockSystem::now()); _memoryFlushConfigUpdater = std::make_unique(memoryFlush, flush.memory); _diskMemUsageSampler->notifier().addDiskMemUsageListener(_memoryFlushConfigUpdater.get()); @@ -290,14 +300,12 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::string fileConfigId; _warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024)); - // We need at least 1 guaranteed free worker in order to ensure progress so #documentsdbs + 1 should suffice, - // but we will not be cheap and give #documentsdbs * 2 - const size_t summaryThreads = std::max(size_t(protonConfig.summary.log.numthreads), protonConfig.documentdb.size() * 2); + + const size_t summaryThreads = deriveBackgroundThreads(protonConfig); _summaryExecutor.reset(new vespalib::BlockingThreadStackExecutor(summaryThreads, 128*1024, summaryThreads*16)); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { - initializeThreads = std::make_shared - (protonConfig.initialize.threads, 128 * 1024); + initializeThreads = std::make_shared(protonConfig.initialize.threads, 128 * 1024); _initDocumentDbsInSequence = (protonConfig.initialize.threads == 1); } _protonConfigurer.applyInitialConfig(initializeThreads); @@ -563,10 +571,8 @@ Proton::addDocumentDB(const document::DocumentType &docType, try { ret->start(); } catch (vespalib::Exception &e) { - LOG(warning, - "Failed to start database for document type '%s'; %s", - docTypeName.toString().c_str(), - e.what()); + LOG(warning, "Failed to start database for document type '%s'; %s", + docTypeName.toString().c_str(), e.what()); return DocumentDB::SP(); } // Wait for replay done on document dbs added due to reconfigs, since engines are already up and running. @@ -577,14 +583,12 @@ Proton::addDocumentDB(const document::DocumentType &docType, _metricsEngine->addDocumentDBMetrics(ret->getMetricsCollection()); _metricsEngine->addMetricsHook(ret->getMetricsUpdateHook()); _documentDBMap[docTypeName] = ret; - if (_persistenceEngine.get() != NULL) { + if (_persistenceEngine) { // Not allowed to get to service layer to call pause(). std::unique_lock persistenceWGuard(_persistenceEngine->getWLock()); - PersistenceHandlerProxy::SP - persistenceHandler(new PersistenceHandlerProxy(ret)); + auto persistenceHandler = std::make_shared(ret); if (!_isInitializing) { - _persistenceEngine-> - propagateSavedClusterState(*persistenceHandler); + _persistenceEngine->propagateSavedClusterState(*persistenceHandler); _persistenceEngine->populateInitialBucketDB(bucketSpace, *persistenceHandler); } // TODO: Fix race with new cluster state setting. @@ -649,10 +653,7 @@ Proton::ping(MonitorRequest::UP request, MonitorClient & client) BootstrapConfig::SP configSnapshot = getActiveConfigSnapshot(); const ProtonConfig &protonConfig = configSnapshot->getProtonConfig(); ret.partid = protonConfig.partition; - if (_matchEngine->isOnline()) - ret.timestamp = 42; // change to flush caches on tld/qrs - else - ret.timestamp = 0; + ret.timestamp = (_matchEngine->isOnline()) ? 42 : 0; ret.activeDocs = getNumActiveDocs(); ret.activeDocsRequested = request->reportActiveDocs; return reply; @@ -661,7 +662,7 @@ Proton::ping(MonitorRequest::UP request, MonitorClient & client) bool Proton::triggerFlush() { - if ((_flushEngine.get() == NULL) || ! _flushEngine->HasThread()) { + if (!_flushEngine || ! _flushEngine->HasThread()) { return false; } _flushEngine->triggerFlush(); @@ -682,13 +683,12 @@ createPrepareRestartConfig(const ProtonConfig &protonConfig) bool Proton::prepareRestart() { - if ((_flushEngine.get() == NULL) || ! _flushEngine->HasThread()) { + if (!_flushEngine || ! _flushEngine->HasThread()) { return false; } BootstrapConfig::SP configSnapshot = getActiveConfigSnapshot(); - IFlushStrategy::SP strategy = - std::make_shared( - createPrepareRestartConfig(configSnapshot->getProtonConfig())); + auto strategy = std::make_shared( + createPrepareRestartConfig(configSnapshot->getProtonConfig())); _flushEngine->setStrategy(strategy); return true; } @@ -700,7 +700,7 @@ int countOpenFiles() static const char * const fd_dir_name = "/proc/self/fd"; int count = 0; DIR *dp = opendir(fd_dir_name); - if (dp != NULL) { + if (dp != nullptr) { struct dirent *ptr; while ((ptr = readdir(dp)) != nullptr) { if (strcmp(".", ptr->d_name) == 0) continue; @@ -738,10 +738,6 @@ Proton::updateMetrics(const vespalib::MonitorGuard &) } } -namespace { -const std::string config_id_tag = "CONFIG ID"; -} // namespace - void Proton::waitForInitDone() { diff --git a/searchlib/src/apps/docstore/benchmarkdatastore.cpp b/searchlib/src/apps/docstore/benchmarkdatastore.cpp index 3dcfbbb31cc..620a139d451 100644 --- a/searchlib/src/apps/docstore/benchmarkdatastore.cpp +++ b/searchlib/src/apps/docstore/benchmarkdatastore.cpp @@ -99,7 +99,7 @@ BenchmarkDataStoreApp::benchmark(const vespalib::string & dir, size_t numReads, tuning._randRead.setWantMemoryMap(); } search::index::DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); + vespalib::ThreadStackExecutor executor(1, 128*1024); transactionlog::NoSyncProxy noTlSyncer; LogDataStore store(executor, dir, config, growStrategy, tuning, fileHeaderContext, diff --git a/searchlib/src/apps/docstore/documentstoreinspect.cpp b/searchlib/src/apps/docstore/documentstoreinspect.cpp index b7f202f1209..40f603c3da1 100644 --- a/searchlib/src/apps/docstore/documentstoreinspect.cpp +++ b/searchlib/src/apps/docstore/documentstoreinspect.cpp @@ -105,7 +105,7 @@ DocumentStoreInspectApp::verify(const vespalib::string & dir) GrowStrategy growStrategy; TuneFileSummary tuning; search::index::DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); + vespalib::ThreadStackExecutor executor(1, 128*1024); transactionlog::NoSyncProxy noTlSyncer; LogDataStore store(executor, dir, config, growStrategy, tuning, diff --git a/searchlib/src/apps/docstore/verifylogdatastore.cpp b/searchlib/src/apps/docstore/verifylogdatastore.cpp index 1fb83248b5b..498516882c1 100644 --- a/searchlib/src/apps/docstore/verifylogdatastore.cpp +++ b/searchlib/src/apps/docstore/verifylogdatastore.cpp @@ -49,7 +49,7 @@ VerifyLogDataStoreApp::verify(const vespalib::string & dir) GrowStrategy growStrategy; TuneFileSummary tuning; search::index::DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); + vespalib::ThreadStackExecutor executor(1, 128*1024); transactionlog::NoSyncProxy noTlSyncer; try { diff --git a/searchlib/src/apps/tests/biglogtest.cpp b/searchlib/src/apps/tests/biglogtest.cpp index b5eb76d40a7..788fb855c7f 100644 --- a/searchlib/src/apps/tests/biglogtest.cpp +++ b/searchlib/src/apps/tests/biglogtest.cpp @@ -134,7 +134,7 @@ factory::factory(std::string dir) : DioTune(), _fileHeaderContext(), _config(), - _executor(_config.getNumThreads(), 128*1024), + _executor(1, 128*1024), _noTlSyncer(), _datastore(_executor, dir, _config, GrowStrategy(), tuning, _fileHeaderContext, _noTlSyncer, NULL) {} diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index b48d754bfb5..2ab24a00557 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -77,7 +77,7 @@ TEST("require that LogDocumentStore::Config equality operator detects inequality EXPECT_TRUE(C() == C()); EXPECT_FALSE(C() != C()); EXPECT_FALSE(C(DC(CompressionConfig::NONE, 100000, 100), LC()) == C()); - EXPECT_FALSE(C(DC(), LC().setNumThreads(7)) == C()); + EXPECT_FALSE(C(DC(), LC().setMaxBucketSpread(7)) == C()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index 7e4b4fa4363..247ee134854 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -234,9 +234,9 @@ Fixture::Fixture() : _baseDir("visitor"), _repo(makeDocTypeRepoConfig()), _storeConfig(DocumentStore::Config(CompressionConfig::NONE, 0, 0), - LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0).setNumThreads(1) + LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0) .setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16384))), - _executor(_storeConfig.getLogConfig().getNumThreads(), 128 * 1024), + _executor(1, 128 * 1024), _fileHeaderContext(), _tlSyncer(), _store(), diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index c7da0c17c80..ed99003c2f7 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -205,9 +205,9 @@ TEST("testGrowing") { LogDataStore::Config config; //(100000, 0.1, 3.0, 0.2, 8, true, CompressionConfig::LZ4, // WriteableFileChunk::Config(CompressionConfig(CompressionConfig::LZ4, 9, 60), 1000)); config.setMaxFileSize(100000).setMaxDiskBloatFactor(0.1).setMaxBucketSpread(3.0).setMinFileSizeFactor(0.2) - .setNumThreads(8).compact2ActiveFile(true).compactCompression({CompressionConfig::LZ4}) + .compact2ActiveFile(true).compactCompression({CompressionConfig::LZ4}) .setFileConfig({{CompressionConfig::LZ4, 9, 60}, 1000}); - vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); + vespalib::ThreadStackExecutor executor(8, 128*1024); DummyFileHeaderContext fileHeaderContext; MyTlSyncer tlSyncer; { @@ -276,30 +276,26 @@ void fetchAndTest(IDataStore & datastore, uint32_t lid, const void *a, size_t sz TEST("testTruncatedIdxFile"){ LogDataStore::Config config; DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); + vespalib::ThreadStackExecutor executor(1, 128*1024); MyTlSyncer tlSyncer; { // Files comes from the 'growing test'. - LogDataStore datastore(executor, - TEST_PATH("bug-7257706"), config, - GrowStrategy(), TuneFileSummary(), - fileHeaderContext, tlSyncer, NULL); + LogDataStore datastore(executor, TEST_PATH("bug-7257706"), config, GrowStrategy(), + TuneFileSummary(), fileHeaderContext, tlSyncer, NULL); EXPECT_EQUAL(354ul, datastore.lastSyncToken()); } const char * magic = "mumbo jumbo"; { - LogDataStore datastore(executor, "bug-7257706-truncated", config, - GrowStrategy(), TuneFileSummary(), - fileHeaderContext, tlSyncer, NULL); + LogDataStore datastore(executor, "bug-7257706-truncated", config, GrowStrategy(), + TuneFileSummary(), fileHeaderContext, tlSyncer, NULL); EXPECT_EQUAL(331ul, datastore.lastSyncToken()); datastore.write(332, 7, magic, strlen(magic)); datastore.write(333, 8, magic, strlen(magic)); datastore.flush(datastore.initFlush(334)); } { - LogDataStore datastore(executor, "bug-7257706-truncated", config, - GrowStrategy(), TuneFileSummary(), - fileHeaderContext, tlSyncer, NULL); + LogDataStore datastore(executor, "bug-7257706-truncated", config, GrowStrategy(), + TuneFileSummary(), fileHeaderContext, tlSyncer, NULL); EXPECT_EQUAL(334ul, datastore.lastSyncToken()); } } @@ -307,7 +303,7 @@ TEST("testTruncatedIdxFile"){ TEST("testThatEmptyIdxFilesAndDanglingDatFilesAreRemoved") { LogDataStore::Config config; DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); + vespalib::ThreadStackExecutor executor(1, 128*1024); MyTlSyncer tlSyncer; LogDataStore datastore(executor, "dangling-test", config, GrowStrategy(), TuneFileSummary(), @@ -320,7 +316,7 @@ TEST("testThatEmptyIdxFilesAndDanglingDatFilesAreRemoved") { TEST("testThatIncompleteCompactedFilesAreRemoved") { LogDataStore::Config config; DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); + vespalib::ThreadStackExecutor executor(1, 128*1024); MyTlSyncer tlSyncer; LogDataStore datastore(executor, "incompletecompact-test", config, GrowStrategy(), TuneFileSummary(), @@ -340,11 +336,10 @@ public: _myDir("visitcache"), _config(), _fileHeaderContext(), - _executor(_config.getNumThreads(), 128*1024), + _executor(1, 128*1024), _tlSyncer(), - _datastore(_executor, _myDir.getDir(), _config, - GrowStrategy(), TuneFileSummary(), - _fileHeaderContext, _tlSyncer, NULL) + _datastore(_executor, _myDir.getDir(), _config, GrowStrategy(), + TuneFileSummary(), _fileHeaderContext, _tlSyncer, NULL) { } ~VisitStore(); IDataStore & getStore() { return _datastore; } @@ -518,10 +513,10 @@ VisitCacheStore::VisitCacheStore() : _myDir("visitcache"), _repo(makeDocTypeRepoConfig()), _config(DocumentStore::Config(CompressionConfig::LZ4, 1000000, 0).allowVisitCaching(true), - LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0).setNumThreads(1) + LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0) .setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16384))), _fileHeaderContext(), - _executor(_config.getLogConfig().getNumThreads(), 128*1024), + _executor(1, 128*1024), _tlSyncer(), _datastore(_executor, _myDir.getDir(), _config, GrowStrategy(), TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr), @@ -596,11 +591,10 @@ TEST("testWriteRead") { { EXPECT_TRUE(FastOS_File::MakeDirectory("empty")); DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); + vespalib::ThreadStackExecutor executor(1, 128*1024); MyTlSyncer tlSyncer; - LogDataStore datastore(executor, "empty", config, - GrowStrategy(), TuneFileSummary(), - fileHeaderContext, tlSyncer, NULL); + LogDataStore datastore(executor, "empty", config, GrowStrategy(), + TuneFileSummary(), fileHeaderContext, tlSyncer, NULL); ASSERT_TRUE(datastore.lastSyncToken() == 0); size_t headerFootprint = datastore.getDiskHeaderFootprint(); EXPECT_LESS(0u, headerFootprint); @@ -633,7 +627,7 @@ TEST("testWriteRead") { } { DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); + vespalib::ThreadStackExecutor executor(1, 128*1024); MyTlSyncer tlSyncer; LogDataStore datastore(executor, "empty", config, GrowStrategy(), TuneFileSummary(), @@ -684,7 +678,7 @@ TEST("requireThatFlushTimeIsAvailableAfterFlush") { fastos::TimeStamp before(fastos::ClockSystem::now()); DummyFileHeaderContext fileHeaderContext; LogDataStore::Config config; - vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024); + vespalib::ThreadStackExecutor executor(1, 128*1024); MyTlSyncer tlSyncer; LogDataStore store(executor, testDir.getDir(), config, GrowStrategy(), TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); @@ -980,7 +974,6 @@ TEST("require that config equality operator detects inequality") { EXPECT_FALSE(C() == C().setMaxDiskBloatFactor(0.3)); EXPECT_FALSE(C() == C().setMaxBucketSpread(0.3)); EXPECT_FALSE(C() == C().setMinFileSizeFactor(0.3)); - EXPECT_FALSE(C() == C().setNumThreads(3)); EXPECT_FALSE(C() == C().setFileConfig(WriteableFileChunk::Config({}, 70))); EXPECT_FALSE(C() == C().disableCrcOnRead(true)); EXPECT_FALSE(C() == C().compact2ActiveFile(false)); diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 6c2509c53c5..4e3178fcdb1 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -34,7 +34,6 @@ LogDataStore::Config::Config() _maxDiskBloatFactor(0.2), _maxBucketSpread(2.5), _minFileSizeFactor(0.2), - _numThreads(8), _skipCrcOnRead(false), _compact2ActiveFile(true), _compactCompression(CompressionConfig::LZ4), @@ -49,7 +48,6 @@ LogDataStore::Config::operator == (const Config & rhs) const { (_minFileSizeFactor == rhs._minFileSizeFactor) && (_compact2ActiveFile == rhs._compact2ActiveFile) && (_skipCrcOnRead == rhs._skipCrcOnRead) && - (_numThreads == rhs._numThreads) && (_compactCompression == rhs._compactCompression) && (_fileConfig == rhs._fileConfig); } diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index 5928602e196..52d78cac2bd 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -43,7 +43,6 @@ public: Config & setMaxBucketSpread(double v) { _maxBucketSpread = v; return *this; } Config & setMinFileSizeFactor(double v) { _minFileSizeFactor = v; return *this; } - Config & setNumThreads(size_t v) { _numThreads = v; return *this; } Config & compactCompression(CompressionConfig v) { _compactCompression = v; return *this; } Config & setFileConfig(WriteableFileChunk::Config v) { _fileConfig = v; return *this; } @@ -52,7 +51,6 @@ public: double getMaxBucketSpread() const { return _maxBucketSpread; } double getMinFileSizeFactor() const { return _minFileSizeFactor; } - size_t getNumThreads() const { return _numThreads; } bool crcOnReadDisabled() const { return _skipCrcOnRead; } bool compact2ActiveFile() const { return _compact2ActiveFile; } const CompressionConfig & compactCompression() const { return _compactCompression; } @@ -67,7 +65,6 @@ public: double _maxDiskBloatFactor; double _maxBucketSpread; double _minFileSizeFactor; - size_t _numThreads; bool _skipCrcOnRead; bool _compact2ActiveFile; CompressionConfig _compactCompression; -- cgit v1.2.3 From a0c814fafbedba6edc8e730dfca1363623bf3437 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 6 Oct 2017 15:41:14 +0200 Subject: summary.log.numthreads -> background.threads. --- .../model/builder/xml/dom/DomSearchTuningBuilder.java | 14 ++++++++++++++ .../com/yahoo/vespa/model/search/NodeFlavorTuning.java | 6 +++--- .../src/main/java/com/yahoo/vespa/model/search/Tuning.java | 13 +++++++++++++ config-model/src/main/resources/schema/content.rnc | 3 +++ .../model/builder/xml/dom/DomSearchTuningBuilderTest.java | 10 ++++++++++ .../com/yahoo/vespa/model/search/NodeFlavorTuningTest.java | 2 +- 6 files changed, 44 insertions(+), 4 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java index ba89169fc4c..4ea638ca41a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java @@ -74,6 +74,8 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui handleSummary(parent, e, t.searchNode); } else if (equals("initialize", e)) { handleInitialize(e, t.searchNode); + } else if (equals("background", e)) { + handleBackground(e, t.searchNode); } } } @@ -251,6 +253,9 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } else if (equals("minfilesizefactor", e)) { s.logStore.minFileSizeFactor = asDouble(e); } else if (equals("numthreads", e)) { + parent.deployLogger().log(Level.WARNING, + "Element 'numthreads is deprecated. Use background.threads instead." + + " For now it will take max of the two."); s.logStore.numThreads = asInt(e); } else if (equals("chunk", e)) { s.logStore.chunk = new Tuning.SearchNode.Summary.Store.Component(true); @@ -268,4 +273,13 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } } + private void handleBackground(Element spec, Tuning.SearchNode sn) { + sn.background = new Tuning.SearchNode.Background(); + for (Element e : XML.getChildren(spec)) { + if (equals("threads", e)) { + sn.background.threads = asInt(e); + } + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java index 5750db08178..6cfd52cc232 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java @@ -27,7 +27,7 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { setHwInfo(builder); tuneDiskWriteSpeed(builder); tuneDocumentStoreMaxFileSize(builder.summary.log); - tuneDocumentStoreNumThreads(builder.summary.log); + tuneDocumentStoreNumThreads(builder.background); tuneFlushStrategyMemoryLimits(builder.flush.memory); tuneFlushStrategyTlsSize(builder.flush.memory); } @@ -56,8 +56,8 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { builder.maxfilesize(fileSizeBytes); } - private void tuneDocumentStoreNumThreads(ProtonConfig.Summary.Log.Builder builder) { - builder.numthreads(max(8, (int)nodeFlavor.getMinCpuCores()/2)); + private void tuneDocumentStoreNumThreads(ProtonConfig.Background.Builder builder) { + builder.threads(max(8, (int)nodeFlavor.getMinCpuCores()/2)); } private void tuneFlushStrategyMemoryLimits(ProtonConfig.Flush.Memory.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index 17af4030cb6..1e3b1783f9b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -334,6 +334,17 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P } } + public static class Background implements ProtonConfig.Producer { + public Integer threads = null; + + @Override + public void getConfig(ProtonConfig.Builder builder) { + if (threads != null) { + builder.background.threads(threads); + } + } + } + public RequestThreads threads = null; public FlushStrategy strategy = null; public Resizing resizing = null; @@ -341,6 +352,7 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P public Attribute attribute = null; public Summary summary = null; public Initialize initialize = null; + public Background background = null; @Override public void getConfig(ProtonConfig.Builder builder) { @@ -351,6 +363,7 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P if (attribute != null) attribute.getConfig(builder); if (summary != null) summary.getConfig(builder); if (initialize != null) initialize.getConfig(builder); + if (background != null) background.getConfig(builder); } } diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 9031e8128df..e75c15ea524 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -353,6 +353,9 @@ Tuning = element tuning { }? & element initialize { element threads { xsd:nonNegativeInteger }? + }? & + element background { + element threads { xsd:nonNegativeInteger }? }? }? } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java index 613827d2cf1..210334a1f23 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java @@ -233,4 +233,14 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest { assertThat(cfg, containsString("initialize.threads 7")); } + @Test + public void requireThatWeCanParseBackgroundTag() { + Tuning t = createTuning(parseXml("", + "7", + "")); + assertEquals(7, t.searchNode.background.threads.intValue()); + String cfg = getProtonCfg(t); + assertThat(cfg, containsString("background.threads 7")); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java index 0e1ad92c70d..75f6abb2542 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java @@ -97,7 +97,7 @@ public class NodeFlavorTuningTest { } private static void assertDocumentStoreNumThreads(int numThreads, double numCores) { - assertEquals(numThreads, configFromNumCoresSetting(numCores).summary().log().numthreads()); + assertEquals(numThreads, configFromNumCoresSetting(numCores).background().threads()); } private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) { -- cgit v1.2.3