diff options
Diffstat (limited to 'searchcore')
20 files changed, 125 insertions, 156 deletions
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 5b50ec49844..5c59b3376ba 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -183,7 +183,7 @@ public: std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - tuneFileDocDB, HwInfo())); + tuneFileDocDB)); mgr.forwardConfig(b); mgr.nextGeneration(0); return DocumentDB::SP( diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index ff393682f9b..dae50fcc860 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -208,7 +208,7 @@ public: std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - _tuneFileDocumentDB, _hwInfo); + _tuneFileDocumentDB); _configMgr.forwardConfig(b); _configMgr.nextGeneration(0); if (! FastOS_File::MakeDirectory((std::string("tmpdb/") + docTypeName).c_str())) { abort(); } diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index e39fd1fe28a..e655ae0d739 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -284,7 +284,7 @@ struct MyConfigSnapshot std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - tuneFileDocumentDB, HwInfo()); + tuneFileDocumentDB); config::DirSpec spec(cfgDir); DocumentDBConfigHelper mgr(spec, "searchdocument"); mgr.forwardConfig(_bootstrap); diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index 9f6eb0ea4e1..f93930d189b 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -99,7 +99,7 @@ Fixture::Fixture() std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - tuneFileDocumentDB, HwInfo())); + tuneFileDocumentDB)); mgr.forwardConfig(b); mgr.nextGeneration(0); _db.reset(new DocumentDB(".", mgr.getConfig(), "tcp/localhost:9014", _queryLimiter, _clock, DocTypeName("typea"), diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index dd35d823b99..6b1031fe558 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -40,12 +40,13 @@ makeBaseConfigSnapshot() DBCM dbcm(spec, "test"); DocumenttypesConfigSP dtcfg(config::ConfigGetter<DocumenttypesConfig>::getConfig("", spec).release()); - BootstrapConfig::SP b(new BootstrapConfig(1, dtcfg, + BootstrapConfig::SP b(new BootstrapConfig(1, + dtcfg, DocumentTypeRepo::SP(new DocumentTypeRepo(*dtcfg)), std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - std::make_shared<TuneFileDocumentDB>(), HwInfo())); + std::make_shared<TuneFileDocumentDB>())); dbcm.forwardConfig(b); dbcm.nextGeneration(0); DocumentDBConfig::SP snap = dbcm.getConfig(); diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 96f742a172c..37e4dfc486d 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -14,12 +14,12 @@ #include <vespa/searchcore/proton/server/proton_config_fetcher.h> #include <vespa/searchcore/proton/server/proton_config_snapshot.h> #include <vespa/searchcore/proton/server/i_proton_configurer.h> -#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/varholder.h> #include <vespa/config-bucketspaces.h> +#include <mutex> using namespace config; using namespace proton; @@ -146,15 +146,14 @@ struct ConfigTestFixture { documenttypesBuilder == bootstrapConfig->getDocumenttypesConfig()); } - BootstrapConfig::SP getBootstrapConfig(int64_t generation, const HwInfo & hwInfo) const { + BootstrapConfig::SP getBootstrapConfig(int64_t generation) const { return BootstrapConfig::SP(new BootstrapConfig(generation, - std::make_shared<DocumenttypesConfig>(documenttypesBuilder), - std::make_shared<DocumentTypeRepo>(documenttypesBuilder), - std::make_shared<ProtonConfig>(protonBuilder), + BootstrapConfig::DocumenttypesConfigSP(new DocumenttypesConfig(documenttypesBuilder)), + DocumentTypeRepo::SP(new DocumentTypeRepo(documenttypesBuilder)), + BootstrapConfig::ProtonConfigSP(new ProtonConfig(protonBuilder)), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(bucketspacesBuilder), - std::make_shared<TuneFileDocumentDB>(), - hwInfo)); + std::make_shared<TuneFileDocumentDB>())); } void reload() { context->reload(); } @@ -233,20 +232,14 @@ TEST_FFF("require that bootstrap config manager updates config", ConfigTestFixtu } DocumentDBConfig::SP -getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr, const HwInfo & hwInfo) +getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr) { ConfigRetriever retriever(mgr.createConfigKeySet(), f.context); - mgr.forwardConfig(f.getBootstrapConfig(1, hwInfo)); + mgr.forwardConfig(f.getBootstrapConfig(1)); mgr.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs return mgr.getConfig(); } -DocumentDBConfig::SP -getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr) -{ - return getDocumentDBConfig(f, mgr, HwInfo()); -} - TEST_FF("require that documentdb config manager subscribes for config", ConfigTestFixture("search"), DocumentDBConfigManager(f1.configId + "/typea", "typea")) { @@ -358,38 +351,4 @@ TEST_FF("require that prune removed documents interval can be set based on age", EXPECT_EQUAL(20, config->getMaintenanceConfigSP()->getPruneRemovedDocumentsConfig().getInterval()); } -TEST_FF("require that docstore config computes cachesize automatically if unset", - ConfigTestFixture("test"), - DocumentDBConfigManager(f1.configId + "/test", "test")) -{ - HwInfo hwInfo(HwInfo::Disk(1, false, false), HwInfo::Memory(1000000), HwInfo::Cpu(1)); - f1.addDocType("test"); - f1.protonBuilder.summary.cache.maxbytes = 2000; - auto config = getDocumentDBConfig(f1, f2, hwInfo); - EXPECT_EQUAL(2000ul, config->getStoreConfig().getMaxCacheBytes()); - - f1.protonBuilder.summary.cache.maxbytes = -7; - config = getDocumentDBConfig(f1, f2, hwInfo); - EXPECT_EQUAL(70000ul, config->getStoreConfig().getMaxCacheBytes()); - - f1.protonBuilder.summary.cache.maxbytes = -700; - config = getDocumentDBConfig(f1, f2, hwInfo); - EXPECT_EQUAL(500000ul, config->getStoreConfig().getMaxCacheBytes()); -} - -TEST("test HwInfo equality") { - EXPECT_TRUE(HwInfo::Cpu(1) == HwInfo::Cpu(1)); - EXPECT_FALSE(HwInfo::Cpu(1) == HwInfo::Cpu(2)); - EXPECT_TRUE(HwInfo::Memory(1) == HwInfo::Memory(1)); - EXPECT_FALSE(HwInfo::Memory(1) == HwInfo::Memory(2)); - EXPECT_TRUE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, false,false)); - EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, false,true)); - EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, true,false)); - EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(2, false,false)); - EXPECT_TRUE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 1ul, 1ul)); - EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 1ul, 2ul)); - EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 2ul, 1ul)); - EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(2, false,false), 1ul, 1ul)); -} - TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index d59a67d73bb..a8336e29f9f 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -171,7 +171,7 @@ struct ConfigFixture { BootstrapConfig::ProtonConfigSP(new ProtonConfig(_protonBuilder)), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(_bucketspacesBuilder), - std::make_shared<TuneFileDocumentDB>(), HwInfo())); + std::make_shared<TuneFileDocumentDB>())); } std::shared_ptr<ProtonConfigSnapshot> getConfigSnapshot() diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index c7a3fc7cb62..d7b1cbfa45f 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -2,7 +2,7 @@ namespace=vespa.config.search.core ## Base directory. The default is ignored as it is assigned by the model -basedir string default="." restart +basedir string default="tmp" restart ## specifies the port number for the persistent internal transport ## protocol provided for a multi-level dispatch system. @@ -210,9 +210,7 @@ grow.numdocs int default=10000 restart grow.multivalueallocfactor double default=0.2 restart ## Control cache size in bytes. -## Postive numbers are absolute in bytes. -## Negative numbers are a percentage of memory. -summary.cache.maxbytes long default=-5 +summary.cache.maxbytes long default=0 ## Include visits in the cache, if the visitoperation allows it. ## This will enable another separate cache of summary.cache.maxbytes size. diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info.h b/searchcore/src/vespa/searchcore/proton/common/hw_info.h index 9129aa43483..cd0ef2817d7 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info.h +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info.h @@ -23,9 +23,6 @@ public: uint64_t sizeBytes() const { return _sizeBytes; } bool slow() const { return _slow; } bool shared() const { return _shared; } - bool operator == (const Disk & rhs) const { - return (_sizeBytes == rhs._sizeBytes) && (_slow == rhs._slow) && (_shared == rhs._shared); - } }; class Memory { @@ -34,7 +31,6 @@ public: public: Memory(uint64_t sizeBytes_) : _sizeBytes(sizeBytes_) {} uint64_t sizeBytes() const { return _sizeBytes; } - bool operator == (const Memory & rhs) const { return _sizeBytes == rhs._sizeBytes; } }; class Cpu { @@ -43,7 +39,6 @@ public: public: Cpu(uint32_t cores_) : _cores(cores_) {} uint32_t cores() const { return _cores; } - bool operator == (const Cpu & rhs) const { return _cores == rhs._cores; } }; private: @@ -71,9 +66,6 @@ public: const Disk &disk() const { return _disk; } const Memory &memory() const { return _memory; } const Cpu &cpu() const { return _cpu; } - bool operator == (const HwInfo & rhs) const { - return (_cpu == rhs._cpu) && (_disk == rhs._disk) && (_memory == rhs._memory); - } }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp index 1bc5cf00ad0..27d430b2893 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp @@ -29,30 +29,29 @@ BootstrapConfig::BootstrapConfig( const ProtonConfigSP &protonConfig, const FiledistributorrpcConfigSP &filedistRpcConfSP, const BucketspacesConfigSP &bucketspaces, - const search::TuneFileDocumentDB::SP &tuneFileDocumentDB, - const HwInfo & hwInfo) + const search::TuneFileDocumentDB::SP &tuneFileDocumentDB) : _documenttypes(documenttypes), _repo(repo), _proton(protonConfig), _fileDistributorRpc(filedistRpcConfSP), _bucketspaces(bucketspaces), _tuneFileDocumentDB(tuneFileDocumentDB), - _hwInfo(hwInfo), _generation(generation) { } -BootstrapConfig::~BootstrapConfig() = default; +BootstrapConfig::~BootstrapConfig() { } bool BootstrapConfig::operator==(const BootstrapConfig &rhs) const { - return equals<DocumenttypesConfig>(_documenttypes.get(), rhs._documenttypes.get()) && + return equals<DocumenttypesConfig>(_documenttypes.get(), + rhs._documenttypes.get()) && _repo.get() == rhs._repo.get() && equals<ProtonConfig>(_proton.get(), rhs._proton.get()) && equals<FiledistributorrpcConfig>(_fileDistributorRpc.get(), rhs._fileDistributorRpc.get()) && equals<BucketspacesConfig>(_bucketspaces.get(), rhs._bucketspaces.get()) && - equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(), rhs._tuneFileDocumentDB.get()) && - (_hwInfo == rhs._hwInfo); + equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(), + rhs._tuneFileDocumentDB.get()); } @@ -62,4 +61,5 @@ BootstrapConfig::valid() const return _documenttypes && _repo && _proton && _fileDistributorRpc && _bucketspaces && _tuneFileDocumentDB; } + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h index 368cc30d8a4..1f888ac1739 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h @@ -3,11 +3,11 @@ #pragma once #include "documentdbconfig.h" -#include <vespa/searchcore/proton/common/hw_info.h> -#include <vespa/searchcore/config/config-proton.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/searchcore/config/config-proton.h> #include <vespa/searchlib/common/tunefileinfo.h> +#include <vespa/config/retriever/configkeyset.h> #include <vespa/config/retriever/configsnapshot.h> #include <vespa/fileacquirer/config-filedistributorrpc.h> @@ -37,7 +37,6 @@ private: FiledistributorrpcConfigSP _fileDistributorRpc; BucketspacesConfigSP _bucketspaces; search::TuneFileDocumentDB::SP _tuneFileDocumentDB; - HwInfo _hwInfo; int64_t _generation; public: @@ -47,21 +46,37 @@ public: const ProtonConfigSP &protonConfig, const FiledistributorrpcConfigSP &filedistRpcConfSP, const BucketspacesConfigSP &bucketspaces, - const search::TuneFileDocumentDB::SP &_tuneFileDocumentDB, - const HwInfo & hwInfo); + const search::TuneFileDocumentDB::SP & + _tuneFileDocumentDB); ~BootstrapConfig(); - const document::DocumenttypesConfig &getDocumenttypesConfig() const { return *_documenttypes; } - const FiledistributorrpcConfig &getFiledistributorrpcConfig() const { return *_fileDistributorRpc; } - const FiledistributorrpcConfigSP &getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; } - const DocumenttypesConfigSP &getDocumenttypesConfigSP() const { return _documenttypes; } - const document::DocumentTypeRepo::SP &getDocumentTypeRepoSP() const { return _repo; } - const vespa::config::search::core::ProtonConfig &getProtonConfig() const { return *_proton; } - const ProtonConfigSP &getProtonConfigSP() const { return _proton; } + const document::DocumenttypesConfig & + getDocumenttypesConfig() const { return *_documenttypes; } + + const cloud::config::filedistribution::FiledistributorrpcConfig & + getFiledistributorrpcConfig() const { return *_fileDistributorRpc; } + + const FiledistributorrpcConfigSP & + getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; } + + const DocumenttypesConfigSP & + getDocumenttypesConfigSP() const { return _documenttypes; } + + const document::DocumentTypeRepo::SP & + getDocumentTypeRepoSP() const { return _repo; } + + const vespa::config::search::core::ProtonConfig & + getProtonConfig() const { return *_proton; } + + const ProtonConfigSP & + getProtonConfigSP() const { return _proton; } + const BucketspacesConfigSP &getBucketspacesConfigSP() const { return _bucketspaces; } - const search::TuneFileDocumentDB::SP &getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; } + + const search::TuneFileDocumentDB::SP & + getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; } + int64_t getGeneration() const { return _generation; } - const HwInfo & getHwInfo() const { return _hwInfo; } /** * Shared pointers are checked for identity, not equality. @@ -71,3 +86,4 @@ public: }; } // namespace proton + diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp index 5459d60e769..9088a0b3b88 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp @@ -2,7 +2,6 @@ #include "bootstrapconfigmanager.h" #include "bootstrapconfig.h" -#include <vespa/searchcore/proton/common/hw_info_sampler.h> #include <vespa/config-bucketspaces.h> #include <vespa/searchlib/common/tunefileinfo.hpp> @@ -39,13 +38,6 @@ BootstrapConfigManager::createConfigKeySet() const .add<BucketspacesConfig>(_configId); } -std::shared_ptr<BootstrapConfig> -BootstrapConfigManager::getConfig() const -{ - std::lock_guard<std::mutex> lock(_pendingConfigMutex); - return _pendingConfigSnapshot; -} - void BootstrapConfigManager::update(const ConfigSnapshot & snapshot) { @@ -91,17 +83,21 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) if (snapshot.isChanged<FiledistributorrpcConfig>(_configId, currentGen)) { LOG(info, "Filedistributorrpc config is changed"); - newFiledistRpcConfSP = snapshot.getConfig<FiledistributorrpcConfig>(_configId); + auto p = snapshot.getConfig<FiledistributorrpcConfig>(_configId); + newFiledistRpcConfSP = BootstrapConfig::FiledistributorrpcConfigSP(std::move(p)); } if (snapshot.isChanged<DocumenttypesConfig>(_configId, currentGen)) { LOG(spam, "Documenttypes config is changed"); - newDocumenttypesConfig = snapshot.getConfig<DocumenttypesConfig>(_configId); - newRepo = std::make_shared<DocumentTypeRepo>(*newDocumenttypesConfig); + std::unique_ptr<DocumenttypesConfig> documenttypesConfig = snapshot.getConfig<DocumenttypesConfig>(_configId); + DocumentTypeRepo::SP repo(new DocumentTypeRepo(*documenttypesConfig)); + newDocumenttypesConfig = DocumenttypesConfigSP(documenttypesConfig.release()); + newRepo = repo; } if (snapshot.isChanged<BucketspacesConfig>(_configId, currentGen)) { LOG(spam, "Bucketspaces config is changed"); - newBucketspacesConfig = snapshot.getConfig<BucketspacesConfig>(_configId); + std::unique_ptr<BucketspacesConfig> bucketspacesConfig = snapshot.getConfig<BucketspacesConfig>(_configId); + newBucketspacesConfig = std::move(bucketspacesConfig); } assert(newProtonConfig); assert(newFiledistRpcConfSP); @@ -110,17 +106,8 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) assert(newDocumenttypesConfig); assert(newRepo); - const ProtonConfig &protonConfig = *newProtonConfig; - const auto &hwDiskCfg = protonConfig.hwinfo.disk; - const auto &hwMemoryCfg = protonConfig.hwinfo.memory; - const auto &hwCpuCfg = protonConfig.hwinfo.cpu; - HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit, - hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores); - HwInfoSampler sampler(protonConfig.basedir, samplerCfg); - auto newSnapshot(std::make_shared<BootstrapConfig>(snapshot.getGeneration(), newDocumenttypesConfig, newRepo, - newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig, - newTuneFileDocumentDB, sampler.hwInfo())); + newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig, newTuneFileDocumentDB)); assert(newSnapshot->valid()); { diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h index 0b0dd341975..3862f11cdd9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h @@ -20,7 +20,12 @@ public: ~BootstrapConfigManager(); const config::ConfigKeySet createConfigKeySet() const; - std::shared_ptr<BootstrapConfig> getConfig() const; + std::shared_ptr<BootstrapConfig> + getConfig() const + { + std::lock_guard<std::mutex> lock(_pendingConfigMutex); + return _pendingConfigSnapshot; + } void update(const config::ConfigSnapshot & snapshot); private: diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index ec22d3293c4..def18b76360 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -2,8 +2,6 @@ #include "documentdbconfigmanager.h" #include "bootstrapconfig.h" -#include <vespa/searchcore/proton/common/hw_info.h> -#include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/config-imported-fields.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-summarymap.h> @@ -12,6 +10,7 @@ #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchsummary/config/config-juniperrc.h> +#include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/time/time_box.h> #include <vespa/log/log.h> @@ -158,17 +157,14 @@ deriveCompression(const T & config) { } DocumentStore::Config -getStoreConfig(const ProtonConfig::Summary::Cache & cache, const HwInfo & hwInfo) +getStoreConfig(const ProtonConfig::Summary::Cache & cache) { - size_t maxBytes = (cache.maxbytes < 0) - ? (hwInfo.memory().sizeBytes()*std::min(50l, -cache.maxbytes))/100l - : cache.maxbytes; - return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching); + return DocumentStore::Config(deriveCompression(cache.compression), cache.maxbytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching); } LogDocumentStore::Config -deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush, const HwInfo & hwInfo) { - DocumentStore::Config config(getStoreConfig(summary.cache, hwInfo)); +deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush) { + DocumentStore::Config config(getStoreConfig(summary.cache)); const ProtonConfig::Summary::Log & log(summary.log); const ProtonConfig::Summary::Log::Chunk & chunk(log.chunk); WriteableFileChunk::Config fileConfig(deriveCompression(chunk.compression), chunk.maxbytes); @@ -181,8 +177,8 @@ deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::M return LogDocumentStore::Config(config, logConfig); } -search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton, const HwInfo & hwInfo) { - return deriveConfig(proton.summary, proton.flush.memory, hwInfo); +search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton) { + return deriveConfig(proton.summary, proton.flush.memory); } using AttributesConfigSP = DocumentDBConfig::AttributesConfigSP; @@ -259,7 +255,10 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) } if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration)) { - newRankProfilesConfig = snapshot.getConfig<RankProfilesConfig>(_configId); + newRankProfilesConfig = + RankProfilesConfigSP( + snapshot.getConfig<RankProfilesConfig>(_configId). + release()); } if (snapshot.isChanged<RankingConstantsConfig>(_configId, currentGeneration)) { RankingConstantsConfigSP newRankingConstantsConfig = RankingConstantsConfigSP( @@ -287,34 +286,34 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) newRankingConstants = std::make_shared<RankingConstants>(constants); } if (snapshot.isChanged<IndexschemaConfig>(_configId, currentGeneration)) { - newIndexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId); + std::unique_ptr<IndexschemaConfig> indexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId); search::index::Schema schema; - search::index::SchemaBuilder::build(*newIndexschemaConfig, schema); + search::index::SchemaBuilder::build(*indexschemaConfig, schema); if (!search::index::SchemaUtil::validateSchema(schema)) { LOG(error, "Cannot use bad index schema, validation failed"); abort(); } + newIndexschemaConfig = IndexschemaConfigSP(indexschemaConfig.release()); } if (snapshot.isChanged<AttributesConfig>(_configId, currentGeneration)) { - newAttributesConfig = snapshot.getConfig<AttributesConfig>(_configId); + newAttributesConfig = AttributesConfigSP(snapshot.getConfig<AttributesConfig>(_configId).release()); } if (snapshot.isChanged<SummaryConfig>(_configId, currentGeneration)) { - newSummaryConfig = snapshot.getConfig<SummaryConfig>(_configId); + newSummaryConfig = SummaryConfigSP(snapshot.getConfig<SummaryConfig>(_configId).release()); } if (snapshot.isChanged<SummarymapConfig>(_configId, currentGeneration)) { - newSummarymapConfig = snapshot.getConfig<SummarymapConfig>(_configId); + newSummarymapConfig = SummarymapConfigSP(snapshot.getConfig<SummarymapConfig>(_configId).release()); } if (snapshot.isChanged<JuniperrcConfig>(_configId, currentGeneration)) { - newJuniperrcConfig = snapshot.getConfig<JuniperrcConfig>(_configId); + newJuniperrcConfig = JuniperrcConfigSP(snapshot.getConfig<JuniperrcConfig>(_configId).release()); } if (snapshot.isChanged<ImportedFieldsConfig>(_configId, currentGeneration)) { - newImportedFieldsConfig = snapshot.getConfig<ImportedFieldsConfig>(_configId); + newImportedFieldsConfig = ImportedFieldsConfigSP(snapshot.getConfig<ImportedFieldsConfig>(_configId).release()); } Schema::SP schema(buildSchema(*newAttributesConfig, *newSummaryConfig, *newIndexschemaConfig)); newMaintenanceConfig = buildMaintenanceConfig(_bootstrapConfig, _docTypeName); - search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig(), - _bootstrapConfig->getHwInfo()); + search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig()); if (newMaintenanceConfig && oldMaintenanceConfig && *newMaintenanceConfig == *oldMaintenanceConfig) { newMaintenanceConfig = oldMaintenanceConfig; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h index bd1cb66de43..80654c7588b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h @@ -2,14 +2,14 @@ #pragma once -#include "documentdbconfig.h" +#include <vespa/vespalib/stllike/string.h> #include <vespa/config/config.h> +#include "documentdbconfig.h" #include <mutex> namespace proton { class BootstrapConfig; - /** * This class manages the subscription for documentdb configs. */ diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index 37a8ee2aadf..cfb96172800 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -2,7 +2,6 @@ #include "fileconfigmanager.h" #include "bootstrapconfig.h" -#include <vespa/searchcore/proton/common/hw_info_sampler.h> #include <vespa/config/print/fileconfigwriter.h> #include <vespa/config/print/fileconfigsnapshotreader.h> #include <vespa/config/print/fileconfigsnapshotwriter.h> @@ -12,7 +11,9 @@ #include <vespa/config-summarymap.h> #include <vespa/config-rank-profiles.h> #include <vespa/searchsummary/config/config-juniperrc.h> +#include <vespa/fastos/file.h> #include <vespa/config/helper/configgetter.hpp> +#include <fstream> #include <sstream> #include <fcntl.h> @@ -359,16 +360,8 @@ FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, * of default values here instead of the current values from the config * server. */ - const ProtonConfig &protonConfig = *_protonConfig; - const auto &hwDiskCfg = protonConfig.hwinfo.disk; - const auto &hwMemoryCfg = protonConfig.hwinfo.memory; - const auto &hwCpuCfg = protonConfig.hwinfo.cpu; - HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit, - hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores); - HwInfoSampler sampler(protonConfig.basedir, samplerCfg); auto bootstrap = std::make_shared<BootstrapConfig>(1, docTypesCfg, repo, _protonConfig, filedistRpcConf, - bucketspaces,currentSnapshot.getTuneFileDocumentDBSP(), - sampler.hwInfo()); + bucketspaces,currentSnapshot.getTuneFileDocumentDBSP()); dbc.forwardConfig(bootstrap); dbc.nextGeneration(0); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 2639958ed70..8ce7e0c79fe 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -202,6 +202,8 @@ Proton::Proton(const config::ConfigUri & configUri, _initStarted(false), _initComplete(false), _initDocumentDbsInSequence(false), + _hwInfo(), + _hwInfoSampler(), _documentDBReferenceRegistry() { _documentDBReferenceRegistry = std::make_shared<DocumentDBReferenceRegistry>(); @@ -229,12 +231,22 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) { assert( _initStarted && ! _initComplete ); const ProtonConfig &protonConfig = configSnapshot->getProtonConfig(); - const HwInfo & hwInfo = configSnapshot->getHwInfo(); - + const auto &hwDiskCfg = protonConfig.hwinfo.disk; + const auto &hwMemoryCfg = protonConfig.hwinfo.memory; + const auto &hwCpuCfg = protonConfig.hwinfo.cpu; + HwInfoSampler::Config samplerCfg(hwDiskCfg.size, + hwDiskCfg.writespeed, + hwDiskCfg.slowwritespeedlimit, + hwDiskCfg.samplewritesize, + hwDiskCfg.shared, + hwMemoryCfg.size, + hwCpuCfg.cores); + _hwInfoSampler = std::make_unique<HwInfoSampler>(protonConfig.basedir, samplerCfg); + _hwInfo = _hwInfoSampler->hwInfo(); setFS4Compression(protonConfig); _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler> (protonConfig.basedir, - diskMemUsageSamplerConfig(protonConfig, hwInfo)); + diskMemUsageSamplerConfig(protonConfig, _hwInfo)); _metricsEngine.reset(new MetricsEngine()); _metricsEngine->addMetricsHook(_metricsHook); @@ -251,8 +263,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) switch (flush.strategy) { case ProtonConfig::Flush::MEMORY: { auto memoryFlush = std::make_shared<MemoryFlush>( - MemoryFlushConfigUpdater::convertConfig(flush.memory, hwInfo.memory()), fastos::ClockSystem::now()); - _memoryFlushConfigUpdater = std::make_unique<MemoryFlushConfigUpdater>(memoryFlush, flush.memory, hwInfo.memory()); + MemoryFlushConfigUpdater::convertConfig(flush.memory, _hwInfo.memory()), fastos::ClockSystem::now()); + _memoryFlushConfigUpdater = std::make_unique<MemoryFlushConfigUpdater>(memoryFlush, flush.memory, _hwInfo.memory()); _diskMemUsageSampler->notifier().addDiskMemUsageListener(_memoryFlushConfigUpdater.get()); strategy = memoryFlush; break; @@ -284,7 +296,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::string fileConfigId; _warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024)); - const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu()); + const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, _hwInfo.cpu()); _summaryExecutor.reset(new vespalib::BlockingThreadStackExecutor(summaryThreads, 128*1024, summaryThreads*16)); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { @@ -339,7 +351,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot) protonConfig.search.memory.limiter.minhits); const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP(); - _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, configSnapshot->getHwInfo())); + _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, _hwInfo)); if (_memoryFlushConfigUpdater) { _memoryFlushConfigUpdater->setConfig(protonConfig.flush.memory); _flushEngine->kick(); @@ -533,7 +545,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, _fileHeaderContext, std::move(config_store), initializeThreads, - bootstrapConfig->getHwInfo())); + _hwInfo)); try { ret->start(); } catch (vespalib::Exception &e) { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 501b5ab8c62..8d1026340ca 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -12,6 +12,7 @@ #include "proton_configurer.h" #include "rpc_hooks.h" #include "bootstrapconfig.h" +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/flushengine/flushengine.h> #include <vespa/searchcore/proton/matchengine/matchengine.h> #include <vespa/searchcore/proton/matching/querylimiter.h> @@ -37,6 +38,7 @@ namespace proton { class DiskMemUsageSampler; +class HwInfoSampler; class IDocumentDBReferenceRegistry; class Proton : public IProtonConfigurerOwner, @@ -122,6 +124,8 @@ private: bool _initStarted; bool _initComplete; bool _initDocumentDbsInSequence; + HwInfo _hwInfo; + std::unique_ptr<HwInfoSampler> _hwInfoSampler; std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry; IDocumentDBConfigOwner * diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp index f2deb456090..deeec695f26 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp @@ -65,7 +65,8 @@ ProtonConfigFetcher::pruneManagerMap(const BootstrapConfig::SP & config) if (_dbManagerMap.find(docTypeName) != _dbManagerMap.end()) { mgr = _dbManagerMap[docTypeName]; } else { - mgr = std::make_shared<DocumentDBConfigManager>(ddb.configid, docTypeName.getName()); + mgr = DocumentDBConfigManager::SP(new DocumentDBConfigManager + (ddb.configid, docTypeName.getName())); } set.add(mgr->createConfigKeySet()); newMap[docTypeName] = mgr; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h index 8ba56deb7e8..c8d1e55e4e4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h @@ -52,9 +52,9 @@ private: using TimePoint = std::chrono::time_point<Clock>; using OldDocumentTypeRepo = std::pair<TimePoint, std::shared_ptr<document::DocumentTypeRepo>>; - BootstrapConfigManager _bootstrapConfigManager; + BootstrapConfigManager _bootstrapConfigManager; config::ConfigRetriever _retriever; - IProtonConfigurer & _owner; + IProtonConfigurer & _owner; mutable std::mutex _mutex; // Protects maps using lock_guard = std::lock_guard<std::mutex>; @@ -71,4 +71,6 @@ private: void rememberDocumentTypeRepo(std::shared_ptr<document::DocumentTypeRepo> repo); }; + } // namespace proton + |