diff options
author | Henning Baldersheim <balder@oath.com> | 2018-01-30 17:59:58 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@oath.com> | 2018-01-30 17:59:58 +0100 |
commit | 0ac738014ccc2eea8bbddf67eee697b6503386c1 (patch) | |
tree | 820a94183eb261971e16b41bb4a055260350e341 | |
parent | 91b5ea5eb3290a305978338f1fe11fa8b738d906 (diff) |
Add the HwInfo to the BootstrapConfig.
19 files changed, 100 insertions, 131 deletions
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 5c59b3376ba..5b50ec49844 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -183,7 +183,7 @@ public: std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - tuneFileDocDB)); + tuneFileDocDB, HwInfo())); mgr.forwardConfig(b); mgr.nextGeneration(0); return DocumentDB::SP( diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index dae50fcc860..ff393682f9b 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -208,7 +208,7 @@ public: std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - _tuneFileDocumentDB); + _tuneFileDocumentDB, _hwInfo); _configMgr.forwardConfig(b); _configMgr.nextGeneration(0); if (! FastOS_File::MakeDirectory((std::string("tmpdb/") + docTypeName).c_str())) { abort(); } diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index e655ae0d739..e39fd1fe28a 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -284,7 +284,7 @@ struct MyConfigSnapshot std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - tuneFileDocumentDB); + tuneFileDocumentDB, HwInfo()); config::DirSpec spec(cfgDir); DocumentDBConfigHelper mgr(spec, "searchdocument"); mgr.forwardConfig(_bootstrap); diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index f93930d189b..9f6eb0ea4e1 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -99,7 +99,7 @@ Fixture::Fixture() std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - tuneFileDocumentDB)); + tuneFileDocumentDB, HwInfo())); mgr.forwardConfig(b); mgr.nextGeneration(0); _db.reset(new DocumentDB(".", mgr.getConfig(), "tcp/localhost:9014", _queryLimiter, _clock, DocTypeName("typea"), diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index 6b1031fe558..dd35d823b99 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -40,13 +40,12 @@ makeBaseConfigSnapshot() DBCM dbcm(spec, "test"); DocumenttypesConfigSP dtcfg(config::ConfigGetter<DocumenttypesConfig>::getConfig("", spec).release()); - BootstrapConfig::SP b(new BootstrapConfig(1, - dtcfg, + BootstrapConfig::SP b(new BootstrapConfig(1, dtcfg, DocumentTypeRepo::SP(new DocumentTypeRepo(*dtcfg)), std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - std::make_shared<TuneFileDocumentDB>())); + std::make_shared<TuneFileDocumentDB>(), HwInfo())); dbcm.forwardConfig(b); dbcm.nextGeneration(0); DocumentDBConfig::SP snap = dbcm.getConfig(); diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 994cbd86aa7..7db1fd001d4 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -60,7 +60,6 @@ struct ConfigTestFixture { ConfigSet set; IConfigContext::SP context; int idcounter; - HwInfo hwInfo; ConfigTestFixture(const std::string & id) : configId(id), @@ -147,14 +146,15 @@ struct ConfigTestFixture { documenttypesBuilder == bootstrapConfig->getDocumenttypesConfig()); } - BootstrapConfig::SP getBootstrapConfig(int64_t generation) const { + BootstrapConfig::SP getBootstrapConfig(int64_t generation, const HwInfo & hwInfo) const { return BootstrapConfig::SP(new BootstrapConfig(generation, - BootstrapConfig::DocumenttypesConfigSP(new DocumenttypesConfig(documenttypesBuilder)), - DocumentTypeRepo::SP(new DocumentTypeRepo(documenttypesBuilder)), - BootstrapConfig::ProtonConfigSP(new ProtonConfig(protonBuilder)), + std::make_shared<DocumenttypesConfig>(documenttypesBuilder), + std::make_shared<DocumentTypeRepo>(documenttypesBuilder), + std::make_shared<ProtonConfig>(protonBuilder), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(bucketspacesBuilder), - std::make_shared<TuneFileDocumentDB>())); + std::make_shared<TuneFileDocumentDB>(), + hwInfo)); } void reload() { context->reload(); } @@ -236,8 +236,8 @@ DocumentDBConfig::SP getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr, const HwInfo & hwInfo) { ConfigRetriever retriever(mgr.createConfigKeySet(), f.context); - mgr.forwardConfig(f.getBootstrapConfig(1)); - mgr.update(retriever.getBootstrapConfigs(), hwInfo); // Cheating, but we only need the configs + mgr.forwardConfig(f.getBootstrapConfig(1, hwInfo)); + mgr.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs return mgr.getConfig(); } @@ -281,7 +281,7 @@ TEST_FF("require that documentdb config manager builds schema with imported attr TEST_FFF("require that proton config fetcher follows changes to bootstrap", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { f3.start(); ASSERT_TRUE(f2._configured); ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig())); @@ -296,7 +296,7 @@ TEST_FFF("require that proton config fetcher follows changes to bootstrap", TEST_FFF("require that proton config fetcher follows changes to doctypes", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { f3.start(); f2._configured = false; @@ -316,7 +316,7 @@ TEST_FFF("require that proton config fetcher follows changes to doctypes", TEST_FFF("require that proton config fetcher reconfigures dbowners", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { f3.start(); ASSERT_FALSE(f2.getDocumentDBConfig("typea")); diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index a8336e29f9f..d59a67d73bb 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -171,7 +171,7 @@ struct ConfigFixture { BootstrapConfig::ProtonConfigSP(new ProtonConfig(_protonBuilder)), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(_bucketspacesBuilder), - std::make_shared<TuneFileDocumentDB>())); + std::make_shared<TuneFileDocumentDB>(), HwInfo())); } std::shared_ptr<ProtonConfigSnapshot> getConfigSnapshot() diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index e223ee64828..c7a3fc7cb62 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -2,7 +2,7 @@ namespace=vespa.config.search.core ## Base directory. The default is ignored as it is assigned by the model -basedir string default="tmp" restart +basedir string default="." restart ## specifies the port number for the persistent internal transport ## protocol provided for a multi-level dispatch system. diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp index 27d430b2893..7838c4810e8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp @@ -29,29 +29,29 @@ BootstrapConfig::BootstrapConfig( const ProtonConfigSP &protonConfig, const FiledistributorrpcConfigSP &filedistRpcConfSP, const BucketspacesConfigSP &bucketspaces, - const search::TuneFileDocumentDB::SP &tuneFileDocumentDB) + const search::TuneFileDocumentDB::SP &tuneFileDocumentDB, + const HwInfo & hwInfo) : _documenttypes(documenttypes), _repo(repo), _proton(protonConfig), _fileDistributorRpc(filedistRpcConfSP), _bucketspaces(bucketspaces), _tuneFileDocumentDB(tuneFileDocumentDB), + _hwInfo(hwInfo), _generation(generation) { } -BootstrapConfig::~BootstrapConfig() { } +BootstrapConfig::~BootstrapConfig() = default; bool BootstrapConfig::operator==(const BootstrapConfig &rhs) const { - return equals<DocumenttypesConfig>(_documenttypes.get(), - rhs._documenttypes.get()) && + return equals<DocumenttypesConfig>(_documenttypes.get(), rhs._documenttypes.get()) && _repo.get() == rhs._repo.get() && equals<ProtonConfig>(_proton.get(), rhs._proton.get()) && equals<FiledistributorrpcConfig>(_fileDistributorRpc.get(), rhs._fileDistributorRpc.get()) && equals<BucketspacesConfig>(_bucketspaces.get(), rhs._bucketspaces.get()) && - equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(), - rhs._tuneFileDocumentDB.get()); + equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(), rhs._tuneFileDocumentDB.get()); } @@ -61,5 +61,4 @@ BootstrapConfig::valid() const return _documenttypes && _repo && _proton && _fileDistributorRpc && _bucketspaces && _tuneFileDocumentDB; } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h index 1e2c269178f..368cc30d8a4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h @@ -3,11 +3,11 @@ #pragma once #include "documentdbconfig.h" +#include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/config/config-proton.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> -#include <vespa/searchcore/config/config-proton.h> #include <vespa/searchlib/common/tunefileinfo.h> -#include <vespa/config/retriever/configkeyset.h> #include <vespa/config/retriever/configsnapshot.h> #include <vespa/fileacquirer/config-filedistributorrpc.h> @@ -37,6 +37,7 @@ private: FiledistributorrpcConfigSP _fileDistributorRpc; BucketspacesConfigSP _bucketspaces; search::TuneFileDocumentDB::SP _tuneFileDocumentDB; + HwInfo _hwInfo; int64_t _generation; public: @@ -46,36 +47,21 @@ public: const ProtonConfigSP &protonConfig, const FiledistributorrpcConfigSP &filedistRpcConfSP, const BucketspacesConfigSP &bucketspaces, - const search::TuneFileDocumentDB::SP &_tuneFileDocumentDB); + const search::TuneFileDocumentDB::SP &_tuneFileDocumentDB, + const HwInfo & hwInfo); ~BootstrapConfig(); - const document::DocumenttypesConfig & - getDocumenttypesConfig() const { return *_documenttypes; } - - const cloud::config::filedistribution::FiledistributorrpcConfig & - getFiledistributorrpcConfig() const { return *_fileDistributorRpc; } - - const FiledistributorrpcConfigSP & - getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; } - - const DocumenttypesConfigSP & - getDocumenttypesConfigSP() const { return _documenttypes; } - - const document::DocumentTypeRepo::SP & - getDocumentTypeRepoSP() const { return _repo; } - - const vespa::config::search::core::ProtonConfig & - getProtonConfig() const { return *_proton; } - - const ProtonConfigSP & - getProtonConfigSP() const { return _proton; } - + const document::DocumenttypesConfig &getDocumenttypesConfig() const { return *_documenttypes; } + const FiledistributorrpcConfig &getFiledistributorrpcConfig() const { return *_fileDistributorRpc; } + const FiledistributorrpcConfigSP &getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; } + const DocumenttypesConfigSP &getDocumenttypesConfigSP() const { return _documenttypes; } + const document::DocumentTypeRepo::SP &getDocumentTypeRepoSP() const { return _repo; } + const vespa::config::search::core::ProtonConfig &getProtonConfig() const { return *_proton; } + const ProtonConfigSP &getProtonConfigSP() const { return _proton; } const BucketspacesConfigSP &getBucketspacesConfigSP() const { return _bucketspaces; } - - const search::TuneFileDocumentDB::SP & - getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; } - + const search::TuneFileDocumentDB::SP &getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; } int64_t getGeneration() const { return _generation; } + const HwInfo & getHwInfo() const { return _hwInfo; } /** * Shared pointers are checked for identity, not equality. @@ -85,4 +71,3 @@ public: }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp index 9088a0b3b88..5459d60e769 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp @@ -2,6 +2,7 @@ #include "bootstrapconfigmanager.h" #include "bootstrapconfig.h" +#include <vespa/searchcore/proton/common/hw_info_sampler.h> #include <vespa/config-bucketspaces.h> #include <vespa/searchlib/common/tunefileinfo.hpp> @@ -38,6 +39,13 @@ BootstrapConfigManager::createConfigKeySet() const .add<BucketspacesConfig>(_configId); } +std::shared_ptr<BootstrapConfig> +BootstrapConfigManager::getConfig() const +{ + std::lock_guard<std::mutex> lock(_pendingConfigMutex); + return _pendingConfigSnapshot; +} + void BootstrapConfigManager::update(const ConfigSnapshot & snapshot) { @@ -83,21 +91,17 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) if (snapshot.isChanged<FiledistributorrpcConfig>(_configId, currentGen)) { LOG(info, "Filedistributorrpc config is changed"); - auto p = snapshot.getConfig<FiledistributorrpcConfig>(_configId); - newFiledistRpcConfSP = BootstrapConfig::FiledistributorrpcConfigSP(std::move(p)); + newFiledistRpcConfSP = snapshot.getConfig<FiledistributorrpcConfig>(_configId); } if (snapshot.isChanged<DocumenttypesConfig>(_configId, currentGen)) { LOG(spam, "Documenttypes config is changed"); - std::unique_ptr<DocumenttypesConfig> documenttypesConfig = snapshot.getConfig<DocumenttypesConfig>(_configId); - DocumentTypeRepo::SP repo(new DocumentTypeRepo(*documenttypesConfig)); - newDocumenttypesConfig = DocumenttypesConfigSP(documenttypesConfig.release()); - newRepo = repo; + newDocumenttypesConfig = snapshot.getConfig<DocumenttypesConfig>(_configId); + newRepo = std::make_shared<DocumentTypeRepo>(*newDocumenttypesConfig); } if (snapshot.isChanged<BucketspacesConfig>(_configId, currentGen)) { LOG(spam, "Bucketspaces config is changed"); - std::unique_ptr<BucketspacesConfig> bucketspacesConfig = snapshot.getConfig<BucketspacesConfig>(_configId); - newBucketspacesConfig = std::move(bucketspacesConfig); + newBucketspacesConfig = snapshot.getConfig<BucketspacesConfig>(_configId); } assert(newProtonConfig); assert(newFiledistRpcConfSP); @@ -106,8 +110,17 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) assert(newDocumenttypesConfig); assert(newRepo); + const ProtonConfig &protonConfig = *newProtonConfig; + const auto &hwDiskCfg = protonConfig.hwinfo.disk; + const auto &hwMemoryCfg = protonConfig.hwinfo.memory; + const auto &hwCpuCfg = protonConfig.hwinfo.cpu; + HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit, + hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores); + HwInfoSampler sampler(protonConfig.basedir, samplerCfg); + auto newSnapshot(std::make_shared<BootstrapConfig>(snapshot.getGeneration(), newDocumenttypesConfig, newRepo, - newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig, newTuneFileDocumentDB)); + newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig, + newTuneFileDocumentDB, sampler.hwInfo())); assert(newSnapshot->valid()); { diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h index 3862f11cdd9..0b0dd341975 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h @@ -20,12 +20,7 @@ public: ~BootstrapConfigManager(); const config::ConfigKeySet createConfigKeySet() const; - std::shared_ptr<BootstrapConfig> - getConfig() const - { - std::lock_guard<std::mutex> lock(_pendingConfigMutex); - return _pendingConfigSnapshot; - } + std::shared_ptr<BootstrapConfig> getConfig() const; void update(const config::ConfigSnapshot & snapshot); private: diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index c10a4ad95fa..ec22d3293c4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -205,7 +205,7 @@ filterImportedAttributes(const AttributesConfigSP &attrCfg) } void -DocumentDBConfigManager::update(const ConfigSnapshot &snapshot, const HwInfo & hwInfo) +DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) { using RankProfilesConfigSP = DocumentDBConfig::RankProfilesConfigSP; using RankingConstantsConfigSP = std::shared_ptr<vespa::config::search::core::RankingConstantsConfig>; @@ -259,10 +259,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot, const HwInfo & h } if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration)) { - newRankProfilesConfig = - RankProfilesConfigSP( - snapshot.getConfig<RankProfilesConfig>(_configId). - release()); + newRankProfilesConfig = snapshot.getConfig<RankProfilesConfig>(_configId); } if (snapshot.isChanged<RankingConstantsConfig>(_configId, currentGeneration)) { RankingConstantsConfigSP newRankingConstantsConfig = RankingConstantsConfigSP( @@ -290,34 +287,34 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot, const HwInfo & h newRankingConstants = std::make_shared<RankingConstants>(constants); } if (snapshot.isChanged<IndexschemaConfig>(_configId, currentGeneration)) { - std::unique_ptr<IndexschemaConfig> indexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId); + newIndexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId); search::index::Schema schema; - search::index::SchemaBuilder::build(*indexschemaConfig, schema); + search::index::SchemaBuilder::build(*newIndexschemaConfig, schema); if (!search::index::SchemaUtil::validateSchema(schema)) { LOG(error, "Cannot use bad index schema, validation failed"); abort(); } - newIndexschemaConfig = IndexschemaConfigSP(indexschemaConfig.release()); } if (snapshot.isChanged<AttributesConfig>(_configId, currentGeneration)) { - newAttributesConfig = AttributesConfigSP(snapshot.getConfig<AttributesConfig>(_configId).release()); + newAttributesConfig = snapshot.getConfig<AttributesConfig>(_configId); } if (snapshot.isChanged<SummaryConfig>(_configId, currentGeneration)) { - newSummaryConfig = SummaryConfigSP(snapshot.getConfig<SummaryConfig>(_configId).release()); + newSummaryConfig = snapshot.getConfig<SummaryConfig>(_configId); } if (snapshot.isChanged<SummarymapConfig>(_configId, currentGeneration)) { - newSummarymapConfig = SummarymapConfigSP(snapshot.getConfig<SummarymapConfig>(_configId).release()); + newSummarymapConfig = snapshot.getConfig<SummarymapConfig>(_configId); } if (snapshot.isChanged<JuniperrcConfig>(_configId, currentGeneration)) { - newJuniperrcConfig = JuniperrcConfigSP(snapshot.getConfig<JuniperrcConfig>(_configId).release()); + newJuniperrcConfig = snapshot.getConfig<JuniperrcConfig>(_configId); } if (snapshot.isChanged<ImportedFieldsConfig>(_configId, currentGeneration)) { - newImportedFieldsConfig = ImportedFieldsConfigSP(snapshot.getConfig<ImportedFieldsConfig>(_configId).release()); + newImportedFieldsConfig = snapshot.getConfig<ImportedFieldsConfig>(_configId); } Schema::SP schema(buildSchema(*newAttributesConfig, *newSummaryConfig, *newIndexschemaConfig)); newMaintenanceConfig = buildMaintenanceConfig(_bootstrapConfig, _docTypeName); - search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig(), hwInfo); + search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig(), + _bootstrapConfig->getHwInfo()); if (newMaintenanceConfig && oldMaintenanceConfig && *newMaintenanceConfig == *oldMaintenanceConfig) { newMaintenanceConfig = oldMaintenanceConfig; } @@ -379,8 +376,7 @@ forwardConfig(const BootstrapConfig::SP & config) } DocumentDBConfigHelper::DocumentDBConfigHelper(const DirSpec &spec, const vespalib::string &docTypeName) - : _hwInfo(std::make_unique<HwInfo>()), - _mgr("", docTypeName), + : _mgr("", docTypeName), _retriever(make_unique<ConfigRetriever>(_mgr.createConfigKeySet(), make_shared<ConfigContext>(spec))) { } @@ -392,7 +388,7 @@ DocumentDBConfigHelper::nextGeneration(int timeoutInMillis) ConfigSnapshot snapshot(_retriever->getBootstrapConfigs(timeoutInMillis)); if (snapshot.empty()) return false; - _mgr.update(snapshot, *_hwInfo); + _mgr.update(snapshot); return true; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h index 44e0c111f47..bd1cb66de43 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h @@ -2,15 +2,13 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <vespa/config/config.h> #include "documentdbconfig.h" +#include <vespa/config/config.h> #include <mutex> namespace proton { class BootstrapConfig; -class HwInfo; /** * This class manages the subscription for documentdb configs. @@ -37,7 +35,7 @@ private: public: DocumentDBConfigManager(const vespalib::string &configId, const vespalib::string &docTypeName); ~DocumentDBConfigManager(); - void update(const config::ConfigSnapshot & snapshot, const HwInfo & hwInfo); + void update(const config::ConfigSnapshot & snapshot); DocumentDBConfig::SP getConfig() const; @@ -59,7 +57,6 @@ public: DocumentDBConfig::SP getConfig() const; void forwardConfig(const std::shared_ptr<BootstrapConfig> & config); private: - std::unique_ptr<HwInfo> _hwInfo; DocumentDBConfigManager _mgr; std::unique_ptr<config::ConfigRetriever> _retriever; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index cfb96172800..37a8ee2aadf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -2,6 +2,7 @@ #include "fileconfigmanager.h" #include "bootstrapconfig.h" +#include <vespa/searchcore/proton/common/hw_info_sampler.h> #include <vespa/config/print/fileconfigwriter.h> #include <vespa/config/print/fileconfigsnapshotreader.h> #include <vespa/config/print/fileconfigsnapshotwriter.h> @@ -11,9 +12,7 @@ #include <vespa/config-summarymap.h> #include <vespa/config-rank-profiles.h> #include <vespa/searchsummary/config/config-juniperrc.h> -#include <vespa/fastos/file.h> #include <vespa/config/helper/configgetter.hpp> -#include <fstream> #include <sstream> #include <fcntl.h> @@ -360,8 +359,16 @@ FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, * of default values here instead of the current values from the config * server. */ + const ProtonConfig &protonConfig = *_protonConfig; + const auto &hwDiskCfg = protonConfig.hwinfo.disk; + const auto &hwMemoryCfg = protonConfig.hwinfo.memory; + const auto &hwCpuCfg = protonConfig.hwinfo.cpu; + HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit, + hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores); + HwInfoSampler sampler(protonConfig.basedir, samplerCfg); auto bootstrap = std::make_shared<BootstrapConfig>(1, docTypesCfg, repo, _protonConfig, filedistRpcConf, - bucketspaces,currentSnapshot.getTuneFileDocumentDBSP()); + bucketspaces,currentSnapshot.getTuneFileDocumentDBSP(), + sampler.hwInfo()); dbc.forwardConfig(bootstrap); dbc.nextGeneration(0); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 28811fef84a..2639958ed70 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -188,8 +188,7 @@ Proton::Proton(const config::ConfigUri & configUri, // serializing startup. _executor(1, 128 * 1024), _protonConfigurer(_executor, *this), - _hwInfo(), - _protonConfigFetcher(configUri, _hwInfo, _protonConfigurer, subscribeTimeout), + _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout), _warmupExecutor(), _summaryExecutor(), _queryLimiter(), @@ -203,7 +202,6 @@ Proton::Proton(const config::ConfigUri & configUri, _initStarted(false), _initComplete(false), _initDocumentDbsInSequence(false), - _hwInfoSampler(), _documentDBReferenceRegistry() { _documentDBReferenceRegistry = std::make_shared<DocumentDBReferenceRegistry>(); @@ -231,22 +229,12 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) { assert( _initStarted && ! _initComplete ); const ProtonConfig &protonConfig = configSnapshot->getProtonConfig(); - const auto &hwDiskCfg = protonConfig.hwinfo.disk; - const auto &hwMemoryCfg = protonConfig.hwinfo.memory; - const auto &hwCpuCfg = protonConfig.hwinfo.cpu; - HwInfoSampler::Config samplerCfg(hwDiskCfg.size, - hwDiskCfg.writespeed, - hwDiskCfg.slowwritespeedlimit, - hwDiskCfg.samplewritesize, - hwDiskCfg.shared, - hwMemoryCfg.size, - hwCpuCfg.cores); - _hwInfoSampler = std::make_unique<HwInfoSampler>(protonConfig.basedir, samplerCfg); - _hwInfo = _hwInfoSampler->hwInfo(); + const HwInfo & hwInfo = configSnapshot->getHwInfo(); + setFS4Compression(protonConfig); _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler> (protonConfig.basedir, - diskMemUsageSamplerConfig(protonConfig, _hwInfo)); + diskMemUsageSamplerConfig(protonConfig, hwInfo)); _metricsEngine.reset(new MetricsEngine()); _metricsEngine->addMetricsHook(_metricsHook); @@ -263,8 +251,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) switch (flush.strategy) { case ProtonConfig::Flush::MEMORY: { auto memoryFlush = std::make_shared<MemoryFlush>( - MemoryFlushConfigUpdater::convertConfig(flush.memory, _hwInfo.memory()), fastos::ClockSystem::now()); - _memoryFlushConfigUpdater = std::make_unique<MemoryFlushConfigUpdater>(memoryFlush, flush.memory, _hwInfo.memory()); + MemoryFlushConfigUpdater::convertConfig(flush.memory, hwInfo.memory()), fastos::ClockSystem::now()); + _memoryFlushConfigUpdater = std::make_unique<MemoryFlushConfigUpdater>(memoryFlush, flush.memory, hwInfo.memory()); _diskMemUsageSampler->notifier().addDiskMemUsageListener(_memoryFlushConfigUpdater.get()); strategy = memoryFlush; break; @@ -296,7 +284,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::string fileConfigId; _warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024)); - const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, _hwInfo.cpu()); + const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu()); _summaryExecutor.reset(new vespalib::BlockingThreadStackExecutor(summaryThreads, 128*1024, summaryThreads*16)); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { @@ -351,7 +339,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot) protonConfig.search.memory.limiter.minhits); const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP(); - _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, _hwInfo)); + _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, configSnapshot->getHwInfo())); if (_memoryFlushConfigUpdater) { _memoryFlushConfigUpdater->setConfig(protonConfig.flush.memory); _flushEngine->kick(); @@ -545,7 +533,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, _fileHeaderContext, std::move(config_store), initializeThreads, - _hwInfo)); + bootstrapConfig->getHwInfo())); try { ret->start(); } catch (vespalib::Exception &e) { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index e6d7093fd44..501b5ab8c62 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -12,7 +12,6 @@ #include "proton_configurer.h" #include "rpc_hooks.h" #include "bootstrapconfig.h" -#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/flushengine/flushengine.h> #include <vespa/searchcore/proton/matchengine/matchengine.h> #include <vespa/searchcore/proton/matching/querylimiter.h> @@ -38,7 +37,6 @@ namespace proton { class DiskMemUsageSampler; -class HwInfoSampler; class IDocumentDBReferenceRegistry; class Proton : public IProtonConfigurerOwner, @@ -110,7 +108,6 @@ private: TransportServer::UP _fs4Server; vespalib::ThreadStackExecutor _executor; ProtonConfigurer _protonConfigurer; - HwInfo _hwInfo; ProtonConfigFetcher _protonConfigFetcher; std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor; std::unique_ptr<vespalib::ThreadStackExecutorBase> _summaryExecutor; @@ -125,7 +122,6 @@ private: bool _initStarted; bool _initComplete; bool _initDocumentDbsInSequence; - std::unique_ptr<HwInfoSampler> _hwInfoSampler; std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry; IDocumentDBConfigOwner * diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp index f83b3f784f3..f2deb456090 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp @@ -17,11 +17,10 @@ using namespace std::chrono_literals; namespace proton { -ProtonConfigFetcher::ProtonConfigFetcher(const config::ConfigUri & configUri, const HwInfo &hwInfo, IProtonConfigurer &owner, uint64_t subscribeTimeout) +ProtonConfigFetcher::ProtonConfigFetcher(const config::ConfigUri & configUri, IProtonConfigurer &owner, uint64_t subscribeTimeout) : _bootstrapConfigManager(configUri.getConfigId()), _retriever(_bootstrapConfigManager.createConfigKeySet(), configUri.getContext(), subscribeTimeout), _owner(owner), - _hwInfo(hwInfo), _mutex(), _dbManagerMap(), _threadPool(128 * 1024, 1), @@ -66,8 +65,7 @@ ProtonConfigFetcher::pruneManagerMap(const BootstrapConfig::SP & config) if (_dbManagerMap.find(docTypeName) != _dbManagerMap.end()) { mgr = _dbManagerMap[docTypeName]; } else { - mgr = DocumentDBConfigManager::SP(new DocumentDBConfigManager - (ddb.configid, docTypeName.getName())); + mgr = std::make_shared<DocumentDBConfigManager>(ddb.configid, docTypeName.getName()); } set.add(mgr->createConfigKeySet()); newMap[docTypeName] = mgr; @@ -82,7 +80,7 @@ ProtonConfigFetcher::updateDocumentDBConfigs(const BootstrapConfig::SP & bootstr lock_guard guard(_mutex); for (auto & entry : _dbManagerMap) { entry.second->forwardConfig(bootstrapConfig); - entry.second->update(snapshot, _hwInfo); + entry.second->update(snapshot); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h index 7165e364dc5..8ba56deb7e8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h @@ -17,7 +17,6 @@ namespace proton { class BootstrapConfig; class IProtonConfigurer; -class HwInfo; /** * A ProtonConfigFetcher monitors all config in proton and document dbs for change @@ -28,7 +27,7 @@ class ProtonConfigFetcher : public FastOS_Runnable public: using BootstrapConfigSP = std::shared_ptr<BootstrapConfig>; - ProtonConfigFetcher(const config::ConfigUri & configUri, const HwInfo & hwInfo,IProtonConfigurer &owner, uint64_t subscribeTimeout); + ProtonConfigFetcher(const config::ConfigUri & configUri, IProtonConfigurer &owner, uint64_t subscribeTimeout); ~ProtonConfigFetcher(); /** * Get the current config generation. @@ -56,7 +55,6 @@ private: BootstrapConfigManager _bootstrapConfigManager; config::ConfigRetriever _retriever; IProtonConfigurer & _owner; - const HwInfo & _hwInfo; mutable std::mutex _mutex; // Protects maps using lock_guard = std::lock_guard<std::mutex>; @@ -73,6 +71,4 @@ private: void rememberDocumentTypeRepo(std::shared_ptr<document::DocumentTypeRepo> repo); }; - } // namespace proton - |