From 700bd36c212986332d14674c7c7e32af994b097c Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 30 Jan 2018 12:12:25 +0000 Subject: Let cache be computed based on available memory. --- .../proton_config_fetcher_test.cpp | 33 ++++++++++++++++++---- .../searchcore/proton/server/bootstrapconfig.h | 3 +- .../proton/server/documentdbconfigmanager.cpp | 25 ++++++++-------- .../proton/server/documentdbconfigmanager.h | 5 +++- .../src/vespa/searchcore/proton/server/proton.cpp | 4 +-- .../src/vespa/searchcore/proton/server/proton.h | 2 +- .../proton/server/proton_config_fetcher.cpp | 5 ++-- .../proton/server/proton_config_fetcher.h | 8 ++++-- 8 files changed, 57 insertions(+), 28 deletions(-) (limited to 'searchcore') diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 37e4dfc486d..d305cbf532b 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -14,12 +14,12 @@ #include #include #include +#include #include #include #include #include #include -#include using namespace config; using namespace proton; @@ -60,6 +60,7 @@ struct ConfigTestFixture { ConfigSet set; IConfigContext::SP context; int idcounter; + HwInfo hwInfo; ConfigTestFixture(const std::string & id) : configId(id), @@ -232,14 +233,20 @@ TEST_FFF("require that bootstrap config manager updates config", ConfigTestFixtu } DocumentDBConfig::SP -getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr) +getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr, const HwInfo & hwInfo) { ConfigRetriever retriever(mgr.createConfigKeySet(), f.context); mgr.forwardConfig(f.getBootstrapConfig(1)); - mgr.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs + mgr.update(retriever.getBootstrapConfigs(), hwInfo); // Cheating, but we only need the configs return mgr.getConfig(); } +DocumentDBConfig::SP +getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr) +{ + return getDocumentDBConfig(f, mgr, HwInfo()); +} + TEST_FF("require that documentdb config manager subscribes for config", ConfigTestFixture("search"), DocumentDBConfigManager(f1.configId + "/typea", "typea")) { @@ -274,7 +281,7 @@ TEST_FF("require that documentdb config manager builds schema with imported attr TEST_FFF("require that proton config fetcher follows changes to bootstrap", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { f3.start(); ASSERT_TRUE(f2._configured); ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig())); @@ -289,7 +296,7 @@ TEST_FFF("require that proton config fetcher follows changes to bootstrap", TEST_FFF("require that proton config fetcher follows changes to doctypes", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { f3.start(); f2._configured = false; @@ -309,7 +316,7 @@ TEST_FFF("require that proton config fetcher follows changes to doctypes", TEST_FFF("require that proton config fetcher reconfigures dbowners", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { f3.start(); ASSERT_FALSE(f2.getDocumentDBConfig("typea")); @@ -351,4 +358,18 @@ TEST_FF("require that prune removed documents interval can be set based on age", EXPECT_EQUAL(20, config->getMaintenanceConfigSP()->getPruneRemovedDocumentsConfig().getInterval()); } +TEST_FF("require that docstore config computes cachesize automatically if unset", + ConfigTestFixture("test"), + DocumentDBConfigManager(f1.configId + "/test", "test")) +{ + HwInfo hwInfo(HwInfo::Disk(1, false, false), HwInfo::Memory(1000000), HwInfo::Cpu(1)); + f1.addDocType("test"); + f1.protonBuilder.summary.cache.maxbytes = 2000; + auto config = getDocumentDBConfig(f1, f2, hwInfo); + EXPECT_EQUAL(2000ul, config->getStoreConfig().getMaxCacheBytes()); + f1.protonBuilder.summary.cache.maxbytes = -1; + config = getDocumentDBConfig(f1, f2, hwInfo); + EXPECT_EQUAL(50000ul, config->getStoreConfig().getMaxCacheBytes()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h index 1f888ac1739..1e2c269178f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h @@ -46,8 +46,7 @@ public: const ProtonConfigSP &protonConfig, const FiledistributorrpcConfigSP &filedistRpcConfSP, const BucketspacesConfigSP &bucketspaces, - const search::TuneFileDocumentDB::SP & - _tuneFileDocumentDB); + const search::TuneFileDocumentDB::SP &_tuneFileDocumentDB); ~BootstrapConfig(); const document::DocumenttypesConfig & diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index def18b76360..27bbd019ff5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -2,6 +2,8 @@ #include "documentdbconfigmanager.h" #include "bootstrapconfig.h" +#include +#include #include #include #include @@ -10,7 +12,6 @@ #include #include #include -#include #include #include @@ -157,14 +158,15 @@ deriveCompression(const T & config) { } DocumentStore::Config -getStoreConfig(const ProtonConfig::Summary::Cache & cache) +getStoreConfig(const ProtonConfig::Summary::Cache & cache, const HwInfo & hwInfo) { - return DocumentStore::Config(deriveCompression(cache.compression), cache.maxbytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching); + size_t maxBytes = (cache.maxbytes < 0) ? hwInfo.memory().sizeBytes()*0.05 : cache.maxbytes; + return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching); } LogDocumentStore::Config -deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush) { - DocumentStore::Config config(getStoreConfig(summary.cache)); +deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush, const HwInfo & hwInfo) { + DocumentStore::Config config(getStoreConfig(summary.cache, hwInfo)); const ProtonConfig::Summary::Log & log(summary.log); const ProtonConfig::Summary::Log::Chunk & chunk(log.chunk); WriteableFileChunk::Config fileConfig(deriveCompression(chunk.compression), chunk.maxbytes); @@ -177,8 +179,8 @@ deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::M return LogDocumentStore::Config(config, logConfig); } -search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton) { - return deriveConfig(proton.summary, proton.flush.memory); +search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton, const HwInfo & hwInfo) { + return deriveConfig(proton.summary, proton.flush.memory, hwInfo); } using AttributesConfigSP = DocumentDBConfig::AttributesConfigSP; @@ -201,7 +203,7 @@ filterImportedAttributes(const AttributesConfigSP &attrCfg) } void -DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) +DocumentDBConfigManager::update(const ConfigSnapshot &snapshot, const HwInfo & hwInfo) { using RankProfilesConfigSP = DocumentDBConfig::RankProfilesConfigSP; using RankingConstantsConfigSP = std::shared_ptr; @@ -313,7 +315,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) Schema::SP schema(buildSchema(*newAttributesConfig, *newSummaryConfig, *newIndexschemaConfig)); newMaintenanceConfig = buildMaintenanceConfig(_bootstrapConfig, _docTypeName); - search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig()); + search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig(), hwInfo); if (newMaintenanceConfig && oldMaintenanceConfig && *newMaintenanceConfig == *oldMaintenanceConfig) { newMaintenanceConfig = oldMaintenanceConfig; } @@ -375,7 +377,8 @@ forwardConfig(const BootstrapConfig::SP & config) } DocumentDBConfigHelper::DocumentDBConfigHelper(const DirSpec &spec, const vespalib::string &docTypeName) - : _mgr("", docTypeName), + : _hwInfo(std::make_unique()), + _mgr("", docTypeName), _retriever(make_unique(_mgr.createConfigKeySet(), make_shared(spec))) { } @@ -387,7 +390,7 @@ DocumentDBConfigHelper::nextGeneration(int timeoutInMillis) ConfigSnapshot snapshot(_retriever->getBootstrapConfigs(timeoutInMillis)); if (snapshot.empty()) return false; - _mgr.update(snapshot); + _mgr.update(snapshot, *_hwInfo); return true; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h index 80654c7588b..44e0c111f47 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h @@ -10,6 +10,8 @@ namespace proton { class BootstrapConfig; +class HwInfo; + /** * This class manages the subscription for documentdb configs. */ @@ -35,7 +37,7 @@ private: public: DocumentDBConfigManager(const vespalib::string &configId, const vespalib::string &docTypeName); ~DocumentDBConfigManager(); - void update(const config::ConfigSnapshot & snapshot); + void update(const config::ConfigSnapshot & snapshot, const HwInfo & hwInfo); DocumentDBConfig::SP getConfig() const; @@ -57,6 +59,7 @@ public: DocumentDBConfig::SP getConfig() const; void forwardConfig(const std::shared_ptr & config); private: + std::unique_ptr _hwInfo; DocumentDBConfigManager _mgr; std::unique_ptr _retriever; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 8ce7e0c79fe..28811fef84a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -188,7 +188,8 @@ Proton::Proton(const config::ConfigUri & configUri, // serializing startup. _executor(1, 128 * 1024), _protonConfigurer(_executor, *this), - _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout), + _hwInfo(), + _protonConfigFetcher(configUri, _hwInfo, _protonConfigurer, subscribeTimeout), _warmupExecutor(), _summaryExecutor(), _queryLimiter(), @@ -202,7 +203,6 @@ Proton::Proton(const config::ConfigUri & configUri, _initStarted(false), _initComplete(false), _initDocumentDbsInSequence(false), - _hwInfo(), _hwInfoSampler(), _documentDBReferenceRegistry() { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 8d1026340ca..e6d7093fd44 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -110,6 +110,7 @@ private: TransportServer::UP _fs4Server; vespalib::ThreadStackExecutor _executor; ProtonConfigurer _protonConfigurer; + HwInfo _hwInfo; ProtonConfigFetcher _protonConfigFetcher; std::unique_ptr _warmupExecutor; std::unique_ptr _summaryExecutor; @@ -124,7 +125,6 @@ private: bool _initStarted; bool _initComplete; bool _initDocumentDbsInSequence; - HwInfo _hwInfo; std::unique_ptr _hwInfoSampler; std::shared_ptr _documentDBReferenceRegistry; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp index deeec695f26..f83b3f784f3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp @@ -17,10 +17,11 @@ using namespace std::chrono_literals; namespace proton { -ProtonConfigFetcher::ProtonConfigFetcher(const config::ConfigUri & configUri, IProtonConfigurer &owner, uint64_t subscribeTimeout) +ProtonConfigFetcher::ProtonConfigFetcher(const config::ConfigUri & configUri, const HwInfo &hwInfo, IProtonConfigurer &owner, uint64_t subscribeTimeout) : _bootstrapConfigManager(configUri.getConfigId()), _retriever(_bootstrapConfigManager.createConfigKeySet(), configUri.getContext(), subscribeTimeout), _owner(owner), + _hwInfo(hwInfo), _mutex(), _dbManagerMap(), _threadPool(128 * 1024, 1), @@ -81,7 +82,7 @@ ProtonConfigFetcher::updateDocumentDBConfigs(const BootstrapConfig::SP & bootstr lock_guard guard(_mutex); for (auto & entry : _dbManagerMap) { entry.second->forwardConfig(bootstrapConfig); - entry.second->update(snapshot); + entry.second->update(snapshot, _hwInfo); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h index c8d1e55e4e4..7165e364dc5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h @@ -17,6 +17,7 @@ namespace proton { class BootstrapConfig; class IProtonConfigurer; +class HwInfo; /** * A ProtonConfigFetcher monitors all config in proton and document dbs for change @@ -27,7 +28,7 @@ class ProtonConfigFetcher : public FastOS_Runnable public: using BootstrapConfigSP = std::shared_ptr; - ProtonConfigFetcher(const config::ConfigUri & configUri, IProtonConfigurer &owner, uint64_t subscribeTimeout); + ProtonConfigFetcher(const config::ConfigUri & configUri, const HwInfo & hwInfo,IProtonConfigurer &owner, uint64_t subscribeTimeout); ~ProtonConfigFetcher(); /** * Get the current config generation. @@ -52,9 +53,10 @@ private: using TimePoint = std::chrono::time_point; using OldDocumentTypeRepo = std::pair>; - BootstrapConfigManager _bootstrapConfigManager; + BootstrapConfigManager _bootstrapConfigManager; config::ConfigRetriever _retriever; - IProtonConfigurer & _owner; + IProtonConfigurer & _owner; + const HwInfo & _hwInfo; mutable std::mutex _mutex; // Protects maps using lock_guard = std::lock_guard; -- cgit v1.2.3 From 91b5ea5eb3290a305978338f1fe11fa8b738d906 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 30 Jan 2018 12:35:06 +0000 Subject: Set a default summary cache of 5% of memory --- .../proton/proton_config_fetcher/proton_config_fetcher_test.cpp | 9 +++++++-- searchcore/src/vespa/searchcore/config/proton.def | 4 +++- .../vespa/searchcore/proton/server/documentdbconfigmanager.cpp | 4 +++- 3 files changed, 13 insertions(+), 4 deletions(-) (limited to 'searchcore') diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index d305cbf532b..994cbd86aa7 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -367,9 +367,14 @@ TEST_FF("require that docstore config computes cachesize automatically if unset" f1.protonBuilder.summary.cache.maxbytes = 2000; auto config = getDocumentDBConfig(f1, f2, hwInfo); EXPECT_EQUAL(2000ul, config->getStoreConfig().getMaxCacheBytes()); - f1.protonBuilder.summary.cache.maxbytes = -1; + + f1.protonBuilder.summary.cache.maxbytes = -7; + config = getDocumentDBConfig(f1, f2, hwInfo); + EXPECT_EQUAL(70000ul, config->getStoreConfig().getMaxCacheBytes()); + + f1.protonBuilder.summary.cache.maxbytes = -700; config = getDocumentDBConfig(f1, f2, hwInfo); - EXPECT_EQUAL(50000ul, config->getStoreConfig().getMaxCacheBytes()); + EXPECT_EQUAL(500000ul, config->getStoreConfig().getMaxCacheBytes()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index d7b1cbfa45f..e223ee64828 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -210,7 +210,9 @@ grow.numdocs int default=10000 restart grow.multivalueallocfactor double default=0.2 restart ## Control cache size in bytes. -summary.cache.maxbytes long default=0 +## Postive numbers are absolute in bytes. +## Negative numbers are a percentage of memory. +summary.cache.maxbytes long default=-5 ## Include visits in the cache, if the visitoperation allows it. ## This will enable another separate cache of summary.cache.maxbytes size. diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 27bbd019ff5..c10a4ad95fa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -160,7 +160,9 @@ deriveCompression(const T & config) { DocumentStore::Config getStoreConfig(const ProtonConfig::Summary::Cache & cache, const HwInfo & hwInfo) { - size_t maxBytes = (cache.maxbytes < 0) ? hwInfo.memory().sizeBytes()*0.05 : cache.maxbytes; + size_t maxBytes = (cache.maxbytes < 0) + ? (hwInfo.memory().sizeBytes()*std::min(50l, -cache.maxbytes))/100l + : cache.maxbytes; return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching); } -- cgit v1.2.3 From 0ac738014ccc2eea8bbddf67eee697b6503386c1 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 30 Jan 2018 17:59:58 +0100 Subject: Add the HwInfo to the BootstrapConfig. --- .../src/apps/tests/persistenceconformance_test.cpp | 2 +- .../src/tests/proton/docsummary/docsummary.cpp | 2 +- .../document_subdbs/document_subdbs_test.cpp | 2 +- .../tests/proton/documentdb/documentdb_test.cpp | 2 +- .../fileconfigmanager/fileconfigmanager_test.cpp | 5 +-- .../proton_config_fetcher_test.cpp | 22 +++++------ .../proton_configurer/proton_configurer_test.cpp | 2 +- searchcore/src/vespa/searchcore/config/proton.def | 2 +- .../searchcore/proton/server/bootstrapconfig.cpp | 13 +++---- .../searchcore/proton/server/bootstrapconfig.h | 43 +++++++--------------- .../proton/server/bootstrapconfigmanager.cpp | 31 +++++++++++----- .../proton/server/bootstrapconfigmanager.h | 7 +--- .../proton/server/documentdbconfigmanager.cpp | 30 +++++++-------- .../proton/server/documentdbconfigmanager.h | 7 +--- .../searchcore/proton/server/fileconfigmanager.cpp | 13 +++++-- .../src/vespa/searchcore/proton/server/proton.cpp | 30 +++++---------- .../src/vespa/searchcore/proton/server/proton.h | 4 -- .../proton/server/proton_config_fetcher.cpp | 8 ++-- .../proton/server/proton_config_fetcher.h | 6 +-- 19 files changed, 100 insertions(+), 131 deletions(-) (limited to 'searchcore') diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 5c59b3376ba..5b50ec49844 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -183,7 +183,7 @@ public: std::make_shared(), std::make_shared(), std::make_shared(), - tuneFileDocDB)); + tuneFileDocDB, HwInfo())); mgr.forwardConfig(b); mgr.nextGeneration(0); return DocumentDB::SP( diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index dae50fcc860..ff393682f9b 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -208,7 +208,7 @@ public: std::make_shared(), std::make_shared(), std::make_shared(), - _tuneFileDocumentDB); + _tuneFileDocumentDB, _hwInfo); _configMgr.forwardConfig(b); _configMgr.nextGeneration(0); if (! FastOS_File::MakeDirectory((std::string("tmpdb/") + docTypeName).c_str())) { abort(); } diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index e655ae0d739..e39fd1fe28a 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -284,7 +284,7 @@ struct MyConfigSnapshot std::make_shared(), std::make_shared(), std::make_shared(), - tuneFileDocumentDB); + tuneFileDocumentDB, HwInfo()); config::DirSpec spec(cfgDir); DocumentDBConfigHelper mgr(spec, "searchdocument"); mgr.forwardConfig(_bootstrap); diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index f93930d189b..9f6eb0ea4e1 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -99,7 +99,7 @@ Fixture::Fixture() std::make_shared(), std::make_shared(), std::make_shared(), - tuneFileDocumentDB)); + tuneFileDocumentDB, HwInfo())); mgr.forwardConfig(b); mgr.nextGeneration(0); _db.reset(new DocumentDB(".", mgr.getConfig(), "tcp/localhost:9014", _queryLimiter, _clock, DocTypeName("typea"), diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index 6b1031fe558..dd35d823b99 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -40,13 +40,12 @@ makeBaseConfigSnapshot() DBCM dbcm(spec, "test"); DocumenttypesConfigSP dtcfg(config::ConfigGetter::getConfig("", spec).release()); - BootstrapConfig::SP b(new BootstrapConfig(1, - dtcfg, + BootstrapConfig::SP b(new BootstrapConfig(1, dtcfg, DocumentTypeRepo::SP(new DocumentTypeRepo(*dtcfg)), std::make_shared(), std::make_shared(), std::make_shared(), - std::make_shared())); + std::make_shared(), HwInfo())); dbcm.forwardConfig(b); dbcm.nextGeneration(0); DocumentDBConfig::SP snap = dbcm.getConfig(); diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 994cbd86aa7..7db1fd001d4 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -60,7 +60,6 @@ struct ConfigTestFixture { ConfigSet set; IConfigContext::SP context; int idcounter; - HwInfo hwInfo; ConfigTestFixture(const std::string & id) : configId(id), @@ -147,14 +146,15 @@ struct ConfigTestFixture { documenttypesBuilder == bootstrapConfig->getDocumenttypesConfig()); } - BootstrapConfig::SP getBootstrapConfig(int64_t generation) const { + BootstrapConfig::SP getBootstrapConfig(int64_t generation, const HwInfo & hwInfo) const { return BootstrapConfig::SP(new BootstrapConfig(generation, - BootstrapConfig::DocumenttypesConfigSP(new DocumenttypesConfig(documenttypesBuilder)), - DocumentTypeRepo::SP(new DocumentTypeRepo(documenttypesBuilder)), - BootstrapConfig::ProtonConfigSP(new ProtonConfig(protonBuilder)), + std::make_shared(documenttypesBuilder), + std::make_shared(documenttypesBuilder), + std::make_shared(protonBuilder), std::make_shared(), std::make_shared(bucketspacesBuilder), - std::make_shared())); + std::make_shared(), + hwInfo)); } void reload() { context->reload(); } @@ -236,8 +236,8 @@ DocumentDBConfig::SP getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr, const HwInfo & hwInfo) { ConfigRetriever retriever(mgr.createConfigKeySet(), f.context); - mgr.forwardConfig(f.getBootstrapConfig(1)); - mgr.update(retriever.getBootstrapConfigs(), hwInfo); // Cheating, but we only need the configs + mgr.forwardConfig(f.getBootstrapConfig(1, hwInfo)); + mgr.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs return mgr.getConfig(); } @@ -281,7 +281,7 @@ TEST_FF("require that documentdb config manager builds schema with imported attr TEST_FFF("require that proton config fetcher follows changes to bootstrap", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { f3.start(); ASSERT_TRUE(f2._configured); ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig())); @@ -296,7 +296,7 @@ TEST_FFF("require that proton config fetcher follows changes to bootstrap", TEST_FFF("require that proton config fetcher follows changes to doctypes", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { f3.start(); f2._configured = false; @@ -316,7 +316,7 @@ TEST_FFF("require that proton config fetcher follows changes to doctypes", TEST_FFF("require that proton config fetcher reconfigures dbowners", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { f3.start(); ASSERT_FALSE(f2.getDocumentDBConfig("typea")); diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index a8336e29f9f..d59a67d73bb 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -171,7 +171,7 @@ struct ConfigFixture { BootstrapConfig::ProtonConfigSP(new ProtonConfig(_protonBuilder)), std::make_shared(), std::make_shared(_bucketspacesBuilder), - std::make_shared())); + std::make_shared(), HwInfo())); } std::shared_ptr getConfigSnapshot() diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index e223ee64828..c7a3fc7cb62 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -2,7 +2,7 @@ namespace=vespa.config.search.core ## Base directory. The default is ignored as it is assigned by the model -basedir string default="tmp" restart +basedir string default="." restart ## specifies the port number for the persistent internal transport ## protocol provided for a multi-level dispatch system. diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp index 27d430b2893..7838c4810e8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp @@ -29,29 +29,29 @@ BootstrapConfig::BootstrapConfig( const ProtonConfigSP &protonConfig, const FiledistributorrpcConfigSP &filedistRpcConfSP, const BucketspacesConfigSP &bucketspaces, - const search::TuneFileDocumentDB::SP &tuneFileDocumentDB) + const search::TuneFileDocumentDB::SP &tuneFileDocumentDB, + const HwInfo & hwInfo) : _documenttypes(documenttypes), _repo(repo), _proton(protonConfig), _fileDistributorRpc(filedistRpcConfSP), _bucketspaces(bucketspaces), _tuneFileDocumentDB(tuneFileDocumentDB), + _hwInfo(hwInfo), _generation(generation) { } -BootstrapConfig::~BootstrapConfig() { } +BootstrapConfig::~BootstrapConfig() = default; bool BootstrapConfig::operator==(const BootstrapConfig &rhs) const { - return equals(_documenttypes.get(), - rhs._documenttypes.get()) && + return equals(_documenttypes.get(), rhs._documenttypes.get()) && _repo.get() == rhs._repo.get() && equals(_proton.get(), rhs._proton.get()) && equals(_fileDistributorRpc.get(), rhs._fileDistributorRpc.get()) && equals(_bucketspaces.get(), rhs._bucketspaces.get()) && - equals(_tuneFileDocumentDB.get(), - rhs._tuneFileDocumentDB.get()); + equals(_tuneFileDocumentDB.get(), rhs._tuneFileDocumentDB.get()); } @@ -61,5 +61,4 @@ BootstrapConfig::valid() const return _documenttypes && _repo && _proton && _fileDistributorRpc && _bucketspaces && _tuneFileDocumentDB; } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h index 1e2c269178f..368cc30d8a4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h @@ -3,11 +3,11 @@ #pragma once #include "documentdbconfig.h" +#include +#include #include #include -#include #include -#include #include #include @@ -37,6 +37,7 @@ private: FiledistributorrpcConfigSP _fileDistributorRpc; BucketspacesConfigSP _bucketspaces; search::TuneFileDocumentDB::SP _tuneFileDocumentDB; + HwInfo _hwInfo; int64_t _generation; public: @@ -46,36 +47,21 @@ public: const ProtonConfigSP &protonConfig, const FiledistributorrpcConfigSP &filedistRpcConfSP, const BucketspacesConfigSP &bucketspaces, - const search::TuneFileDocumentDB::SP &_tuneFileDocumentDB); + const search::TuneFileDocumentDB::SP &_tuneFileDocumentDB, + const HwInfo & hwInfo); ~BootstrapConfig(); - const document::DocumenttypesConfig & - getDocumenttypesConfig() const { return *_documenttypes; } - - const cloud::config::filedistribution::FiledistributorrpcConfig & - getFiledistributorrpcConfig() const { return *_fileDistributorRpc; } - - const FiledistributorrpcConfigSP & - getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; } - - const DocumenttypesConfigSP & - getDocumenttypesConfigSP() const { return _documenttypes; } - - const document::DocumentTypeRepo::SP & - getDocumentTypeRepoSP() const { return _repo; } - - const vespa::config::search::core::ProtonConfig & - getProtonConfig() const { return *_proton; } - - const ProtonConfigSP & - getProtonConfigSP() const { return _proton; } - + const document::DocumenttypesConfig &getDocumenttypesConfig() const { return *_documenttypes; } + const FiledistributorrpcConfig &getFiledistributorrpcConfig() const { return *_fileDistributorRpc; } + const FiledistributorrpcConfigSP &getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; } + const DocumenttypesConfigSP &getDocumenttypesConfigSP() const { return _documenttypes; } + const document::DocumentTypeRepo::SP &getDocumentTypeRepoSP() const { return _repo; } + const vespa::config::search::core::ProtonConfig &getProtonConfig() const { return *_proton; } + const ProtonConfigSP &getProtonConfigSP() const { return _proton; } const BucketspacesConfigSP &getBucketspacesConfigSP() const { return _bucketspaces; } - - const search::TuneFileDocumentDB::SP & - getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; } - + const search::TuneFileDocumentDB::SP &getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; } int64_t getGeneration() const { return _generation; } + const HwInfo & getHwInfo() const { return _hwInfo; } /** * Shared pointers are checked for identity, not equality. @@ -85,4 +71,3 @@ public: }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp index 9088a0b3b88..5459d60e769 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp @@ -2,6 +2,7 @@ #include "bootstrapconfigmanager.h" #include "bootstrapconfig.h" +#include #include #include @@ -38,6 +39,13 @@ BootstrapConfigManager::createConfigKeySet() const .add(_configId); } +std::shared_ptr +BootstrapConfigManager::getConfig() const +{ + std::lock_guard lock(_pendingConfigMutex); + return _pendingConfigSnapshot; +} + void BootstrapConfigManager::update(const ConfigSnapshot & snapshot) { @@ -83,21 +91,17 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) if (snapshot.isChanged(_configId, currentGen)) { LOG(info, "Filedistributorrpc config is changed"); - auto p = snapshot.getConfig(_configId); - newFiledistRpcConfSP = BootstrapConfig::FiledistributorrpcConfigSP(std::move(p)); + newFiledistRpcConfSP = snapshot.getConfig(_configId); } if (snapshot.isChanged(_configId, currentGen)) { LOG(spam, "Documenttypes config is changed"); - std::unique_ptr documenttypesConfig = snapshot.getConfig(_configId); - DocumentTypeRepo::SP repo(new DocumentTypeRepo(*documenttypesConfig)); - newDocumenttypesConfig = DocumenttypesConfigSP(documenttypesConfig.release()); - newRepo = repo; + newDocumenttypesConfig = snapshot.getConfig(_configId); + newRepo = std::make_shared(*newDocumenttypesConfig); } if (snapshot.isChanged(_configId, currentGen)) { LOG(spam, "Bucketspaces config is changed"); - std::unique_ptr bucketspacesConfig = snapshot.getConfig(_configId); - newBucketspacesConfig = std::move(bucketspacesConfig); + newBucketspacesConfig = snapshot.getConfig(_configId); } assert(newProtonConfig); assert(newFiledistRpcConfSP); @@ -106,8 +110,17 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) assert(newDocumenttypesConfig); assert(newRepo); + const ProtonConfig &protonConfig = *newProtonConfig; + const auto &hwDiskCfg = protonConfig.hwinfo.disk; + const auto &hwMemoryCfg = protonConfig.hwinfo.memory; + const auto &hwCpuCfg = protonConfig.hwinfo.cpu; + HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit, + hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores); + HwInfoSampler sampler(protonConfig.basedir, samplerCfg); + auto newSnapshot(std::make_shared(snapshot.getGeneration(), newDocumenttypesConfig, newRepo, - newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig, newTuneFileDocumentDB)); + newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig, + newTuneFileDocumentDB, sampler.hwInfo())); assert(newSnapshot->valid()); { diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h index 3862f11cdd9..0b0dd341975 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h @@ -20,12 +20,7 @@ public: ~BootstrapConfigManager(); const config::ConfigKeySet createConfigKeySet() const; - std::shared_ptr - getConfig() const - { - std::lock_guard lock(_pendingConfigMutex); - return _pendingConfigSnapshot; - } + std::shared_ptr getConfig() const; void update(const config::ConfigSnapshot & snapshot); private: diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index c10a4ad95fa..ec22d3293c4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -205,7 +205,7 @@ filterImportedAttributes(const AttributesConfigSP &attrCfg) } void -DocumentDBConfigManager::update(const ConfigSnapshot &snapshot, const HwInfo & hwInfo) +DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) { using RankProfilesConfigSP = DocumentDBConfig::RankProfilesConfigSP; using RankingConstantsConfigSP = std::shared_ptr; @@ -259,10 +259,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot, const HwInfo & h } if (snapshot.isChanged(_configId, currentGeneration)) { - newRankProfilesConfig = - RankProfilesConfigSP( - snapshot.getConfig(_configId). - release()); + newRankProfilesConfig = snapshot.getConfig(_configId); } if (snapshot.isChanged(_configId, currentGeneration)) { RankingConstantsConfigSP newRankingConstantsConfig = RankingConstantsConfigSP( @@ -290,34 +287,34 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot, const HwInfo & h newRankingConstants = std::make_shared(constants); } if (snapshot.isChanged(_configId, currentGeneration)) { - std::unique_ptr indexschemaConfig = snapshot.getConfig(_configId); + newIndexschemaConfig = snapshot.getConfig(_configId); search::index::Schema schema; - search::index::SchemaBuilder::build(*indexschemaConfig, schema); + search::index::SchemaBuilder::build(*newIndexschemaConfig, schema); if (!search::index::SchemaUtil::validateSchema(schema)) { LOG(error, "Cannot use bad index schema, validation failed"); abort(); } - newIndexschemaConfig = IndexschemaConfigSP(indexschemaConfig.release()); } if (snapshot.isChanged(_configId, currentGeneration)) { - newAttributesConfig = AttributesConfigSP(snapshot.getConfig(_configId).release()); + newAttributesConfig = snapshot.getConfig(_configId); } if (snapshot.isChanged(_configId, currentGeneration)) { - newSummaryConfig = SummaryConfigSP(snapshot.getConfig(_configId).release()); + newSummaryConfig = snapshot.getConfig(_configId); } if (snapshot.isChanged(_configId, currentGeneration)) { - newSummarymapConfig = SummarymapConfigSP(snapshot.getConfig(_configId).release()); + newSummarymapConfig = snapshot.getConfig(_configId); } if (snapshot.isChanged(_configId, currentGeneration)) { - newJuniperrcConfig = JuniperrcConfigSP(snapshot.getConfig(_configId).release()); + newJuniperrcConfig = snapshot.getConfig(_configId); } if (snapshot.isChanged(_configId, currentGeneration)) { - newImportedFieldsConfig = ImportedFieldsConfigSP(snapshot.getConfig(_configId).release()); + newImportedFieldsConfig = snapshot.getConfig(_configId); } Schema::SP schema(buildSchema(*newAttributesConfig, *newSummaryConfig, *newIndexschemaConfig)); newMaintenanceConfig = buildMaintenanceConfig(_bootstrapConfig, _docTypeName); - search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig(), hwInfo); + search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig(), + _bootstrapConfig->getHwInfo()); if (newMaintenanceConfig && oldMaintenanceConfig && *newMaintenanceConfig == *oldMaintenanceConfig) { newMaintenanceConfig = oldMaintenanceConfig; } @@ -379,8 +376,7 @@ forwardConfig(const BootstrapConfig::SP & config) } DocumentDBConfigHelper::DocumentDBConfigHelper(const DirSpec &spec, const vespalib::string &docTypeName) - : _hwInfo(std::make_unique()), - _mgr("", docTypeName), + : _mgr("", docTypeName), _retriever(make_unique(_mgr.createConfigKeySet(), make_shared(spec))) { } @@ -392,7 +388,7 @@ DocumentDBConfigHelper::nextGeneration(int timeoutInMillis) ConfigSnapshot snapshot(_retriever->getBootstrapConfigs(timeoutInMillis)); if (snapshot.empty()) return false; - _mgr.update(snapshot, *_hwInfo); + _mgr.update(snapshot); return true; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h index 44e0c111f47..bd1cb66de43 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h @@ -2,15 +2,13 @@ #pragma once -#include -#include #include "documentdbconfig.h" +#include #include namespace proton { class BootstrapConfig; -class HwInfo; /** * This class manages the subscription for documentdb configs. @@ -37,7 +35,7 @@ private: public: DocumentDBConfigManager(const vespalib::string &configId, const vespalib::string &docTypeName); ~DocumentDBConfigManager(); - void update(const config::ConfigSnapshot & snapshot, const HwInfo & hwInfo); + void update(const config::ConfigSnapshot & snapshot); DocumentDBConfig::SP getConfig() const; @@ -59,7 +57,6 @@ public: DocumentDBConfig::SP getConfig() const; void forwardConfig(const std::shared_ptr & config); private: - std::unique_ptr _hwInfo; DocumentDBConfigManager _mgr; std::unique_ptr _retriever; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index cfb96172800..37a8ee2aadf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -2,6 +2,7 @@ #include "fileconfigmanager.h" #include "bootstrapconfig.h" +#include #include #include #include @@ -11,9 +12,7 @@ #include #include #include -#include #include -#include #include #include @@ -360,8 +359,16 @@ FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, * of default values here instead of the current values from the config * server. */ + const ProtonConfig &protonConfig = *_protonConfig; + const auto &hwDiskCfg = protonConfig.hwinfo.disk; + const auto &hwMemoryCfg = protonConfig.hwinfo.memory; + const auto &hwCpuCfg = protonConfig.hwinfo.cpu; + HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit, + hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores); + HwInfoSampler sampler(protonConfig.basedir, samplerCfg); auto bootstrap = std::make_shared(1, docTypesCfg, repo, _protonConfig, filedistRpcConf, - bucketspaces,currentSnapshot.getTuneFileDocumentDBSP()); + bucketspaces,currentSnapshot.getTuneFileDocumentDBSP(), + sampler.hwInfo()); dbc.forwardConfig(bootstrap); dbc.nextGeneration(0); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 28811fef84a..2639958ed70 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -188,8 +188,7 @@ Proton::Proton(const config::ConfigUri & configUri, // serializing startup. _executor(1, 128 * 1024), _protonConfigurer(_executor, *this), - _hwInfo(), - _protonConfigFetcher(configUri, _hwInfo, _protonConfigurer, subscribeTimeout), + _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout), _warmupExecutor(), _summaryExecutor(), _queryLimiter(), @@ -203,7 +202,6 @@ Proton::Proton(const config::ConfigUri & configUri, _initStarted(false), _initComplete(false), _initDocumentDbsInSequence(false), - _hwInfoSampler(), _documentDBReferenceRegistry() { _documentDBReferenceRegistry = std::make_shared(); @@ -231,22 +229,12 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) { assert( _initStarted && ! _initComplete ); const ProtonConfig &protonConfig = configSnapshot->getProtonConfig(); - const auto &hwDiskCfg = protonConfig.hwinfo.disk; - const auto &hwMemoryCfg = protonConfig.hwinfo.memory; - const auto &hwCpuCfg = protonConfig.hwinfo.cpu; - HwInfoSampler::Config samplerCfg(hwDiskCfg.size, - hwDiskCfg.writespeed, - hwDiskCfg.slowwritespeedlimit, - hwDiskCfg.samplewritesize, - hwDiskCfg.shared, - hwMemoryCfg.size, - hwCpuCfg.cores); - _hwInfoSampler = std::make_unique(protonConfig.basedir, samplerCfg); - _hwInfo = _hwInfoSampler->hwInfo(); + const HwInfo & hwInfo = configSnapshot->getHwInfo(); + setFS4Compression(protonConfig); _diskMemUsageSampler = std::make_unique (protonConfig.basedir, - diskMemUsageSamplerConfig(protonConfig, _hwInfo)); + diskMemUsageSamplerConfig(protonConfig, hwInfo)); _metricsEngine.reset(new MetricsEngine()); _metricsEngine->addMetricsHook(_metricsHook); @@ -263,8 +251,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) switch (flush.strategy) { case ProtonConfig::Flush::MEMORY: { auto memoryFlush = std::make_shared( - MemoryFlushConfigUpdater::convertConfig(flush.memory, _hwInfo.memory()), fastos::ClockSystem::now()); - _memoryFlushConfigUpdater = std::make_unique(memoryFlush, flush.memory, _hwInfo.memory()); + MemoryFlushConfigUpdater::convertConfig(flush.memory, hwInfo.memory()), fastos::ClockSystem::now()); + _memoryFlushConfigUpdater = std::make_unique(memoryFlush, flush.memory, hwInfo.memory()); _diskMemUsageSampler->notifier().addDiskMemUsageListener(_memoryFlushConfigUpdater.get()); strategy = memoryFlush; break; @@ -296,7 +284,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::string fileConfigId; _warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024)); - const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, _hwInfo.cpu()); + const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu()); _summaryExecutor.reset(new vespalib::BlockingThreadStackExecutor(summaryThreads, 128*1024, summaryThreads*16)); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { @@ -351,7 +339,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot) protonConfig.search.memory.limiter.minhits); const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP(); - _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, _hwInfo)); + _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, configSnapshot->getHwInfo())); if (_memoryFlushConfigUpdater) { _memoryFlushConfigUpdater->setConfig(protonConfig.flush.memory); _flushEngine->kick(); @@ -545,7 +533,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, _fileHeaderContext, std::move(config_store), initializeThreads, - _hwInfo)); + bootstrapConfig->getHwInfo())); try { ret->start(); } catch (vespalib::Exception &e) { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index e6d7093fd44..501b5ab8c62 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -12,7 +12,6 @@ #include "proton_configurer.h" #include "rpc_hooks.h" #include "bootstrapconfig.h" -#include #include #include #include @@ -38,7 +37,6 @@ namespace proton { class DiskMemUsageSampler; -class HwInfoSampler; class IDocumentDBReferenceRegistry; class Proton : public IProtonConfigurerOwner, @@ -110,7 +108,6 @@ private: TransportServer::UP _fs4Server; vespalib::ThreadStackExecutor _executor; ProtonConfigurer _protonConfigurer; - HwInfo _hwInfo; ProtonConfigFetcher _protonConfigFetcher; std::unique_ptr _warmupExecutor; std::unique_ptr _summaryExecutor; @@ -125,7 +122,6 @@ private: bool _initStarted; bool _initComplete; bool _initDocumentDbsInSequence; - std::unique_ptr _hwInfoSampler; std::shared_ptr _documentDBReferenceRegistry; IDocumentDBConfigOwner * diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp index f83b3f784f3..f2deb456090 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp @@ -17,11 +17,10 @@ using namespace std::chrono_literals; namespace proton { -ProtonConfigFetcher::ProtonConfigFetcher(const config::ConfigUri & configUri, const HwInfo &hwInfo, IProtonConfigurer &owner, uint64_t subscribeTimeout) +ProtonConfigFetcher::ProtonConfigFetcher(const config::ConfigUri & configUri, IProtonConfigurer &owner, uint64_t subscribeTimeout) : _bootstrapConfigManager(configUri.getConfigId()), _retriever(_bootstrapConfigManager.createConfigKeySet(), configUri.getContext(), subscribeTimeout), _owner(owner), - _hwInfo(hwInfo), _mutex(), _dbManagerMap(), _threadPool(128 * 1024, 1), @@ -66,8 +65,7 @@ ProtonConfigFetcher::pruneManagerMap(const BootstrapConfig::SP & config) if (_dbManagerMap.find(docTypeName) != _dbManagerMap.end()) { mgr = _dbManagerMap[docTypeName]; } else { - mgr = DocumentDBConfigManager::SP(new DocumentDBConfigManager - (ddb.configid, docTypeName.getName())); + mgr = std::make_shared(ddb.configid, docTypeName.getName()); } set.add(mgr->createConfigKeySet()); newMap[docTypeName] = mgr; @@ -82,7 +80,7 @@ ProtonConfigFetcher::updateDocumentDBConfigs(const BootstrapConfig::SP & bootstr lock_guard guard(_mutex); for (auto & entry : _dbManagerMap) { entry.second->forwardConfig(bootstrapConfig); - entry.second->update(snapshot, _hwInfo); + entry.second->update(snapshot); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h index 7165e364dc5..8ba56deb7e8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h @@ -17,7 +17,6 @@ namespace proton { class BootstrapConfig; class IProtonConfigurer; -class HwInfo; /** * A ProtonConfigFetcher monitors all config in proton and document dbs for change @@ -28,7 +27,7 @@ class ProtonConfigFetcher : public FastOS_Runnable public: using BootstrapConfigSP = std::shared_ptr; - ProtonConfigFetcher(const config::ConfigUri & configUri, const HwInfo & hwInfo,IProtonConfigurer &owner, uint64_t subscribeTimeout); + ProtonConfigFetcher(const config::ConfigUri & configUri, IProtonConfigurer &owner, uint64_t subscribeTimeout); ~ProtonConfigFetcher(); /** * Get the current config generation. @@ -56,7 +55,6 @@ private: BootstrapConfigManager _bootstrapConfigManager; config::ConfigRetriever _retriever; IProtonConfigurer & _owner; - const HwInfo & _hwInfo; mutable std::mutex _mutex; // Protects maps using lock_guard = std::lock_guard; @@ -73,6 +71,4 @@ private: void rememberDocumentTypeRepo(std::shared_ptr repo); }; - } // namespace proton - -- cgit v1.2.3 From 395627362aec1f0ee7dc6b5115789f96fbc86dce Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 31 Jan 2018 14:14:11 +0100 Subject: Implement and test HwInfo equality. --- .../proton_config_fetcher/proton_config_fetcher_test.cpp | 15 +++++++++++++++ searchcore/src/vespa/searchcore/proton/common/hw_info.h | 8 ++++++++ .../vespa/searchcore/proton/server/bootstrapconfig.cpp | 3 ++- 3 files changed, 25 insertions(+), 1 deletion(-) (limited to 'searchcore') diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 7db1fd001d4..96f742a172c 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -377,4 +377,19 @@ TEST_FF("require that docstore config computes cachesize automatically if unset" EXPECT_EQUAL(500000ul, config->getStoreConfig().getMaxCacheBytes()); } +TEST("test HwInfo equality") { + EXPECT_TRUE(HwInfo::Cpu(1) == HwInfo::Cpu(1)); + EXPECT_FALSE(HwInfo::Cpu(1) == HwInfo::Cpu(2)); + EXPECT_TRUE(HwInfo::Memory(1) == HwInfo::Memory(1)); + EXPECT_FALSE(HwInfo::Memory(1) == HwInfo::Memory(2)); + EXPECT_TRUE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, false,false)); + EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, false,true)); + EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, true,false)); + EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(2, false,false)); + EXPECT_TRUE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 1ul, 1ul)); + EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 1ul, 2ul)); + EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 2ul, 1ul)); + EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(2, false,false), 1ul, 1ul)); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info.h b/searchcore/src/vespa/searchcore/proton/common/hw_info.h index cd0ef2817d7..9129aa43483 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info.h +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info.h @@ -23,6 +23,9 @@ public: uint64_t sizeBytes() const { return _sizeBytes; } bool slow() const { return _slow; } bool shared() const { return _shared; } + bool operator == (const Disk & rhs) const { + return (_sizeBytes == rhs._sizeBytes) && (_slow == rhs._slow) && (_shared == rhs._shared); + } }; class Memory { @@ -31,6 +34,7 @@ public: public: Memory(uint64_t sizeBytes_) : _sizeBytes(sizeBytes_) {} uint64_t sizeBytes() const { return _sizeBytes; } + bool operator == (const Memory & rhs) const { return _sizeBytes == rhs._sizeBytes; } }; class Cpu { @@ -39,6 +43,7 @@ public: public: Cpu(uint32_t cores_) : _cores(cores_) {} uint32_t cores() const { return _cores; } + bool operator == (const Cpu & rhs) const { return _cores == rhs._cores; } }; private: @@ -66,6 +71,9 @@ public: const Disk &disk() const { return _disk; } const Memory &memory() const { return _memory; } const Cpu &cpu() const { return _cpu; } + bool operator == (const HwInfo & rhs) const { + return (_cpu == rhs._cpu) && (_disk == rhs._disk) && (_memory == rhs._memory); + } }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp index 7838c4810e8..1bc5cf00ad0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp @@ -51,7 +51,8 @@ BootstrapConfig::operator==(const BootstrapConfig &rhs) const equals(_proton.get(), rhs._proton.get()) && equals(_fileDistributorRpc.get(), rhs._fileDistributorRpc.get()) && equals(_bucketspaces.get(), rhs._bucketspaces.get()) && - equals(_tuneFileDocumentDB.get(), rhs._tuneFileDocumentDB.get()); + equals(_tuneFileDocumentDB.get(), rhs._tuneFileDocumentDB.get()) && + (_hwInfo == rhs._hwInfo); } -- cgit v1.2.3