diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2018-01-31 18:42:15 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-01-31 18:42:15 +0100 |
commit | 01dcc51225aed6e017d54875a0c924f0fab7aef1 (patch) | |
tree | f538eb6f83f8a0c0d5df4a0c4d383985a92725ab /searchcore | |
parent | 2b42b7abba86daa12dbc5691d1873bec14a94d00 (diff) |
Revert "Revert "Balder/compute summary cache automatically in backend""
Diffstat (limited to 'searchcore')
20 files changed, 156 insertions, 125 deletions
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 5c59b3376ba..5b50ec49844 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -183,7 +183,7 @@ public: std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - tuneFileDocDB)); + tuneFileDocDB, HwInfo())); mgr.forwardConfig(b); mgr.nextGeneration(0); return DocumentDB::SP( diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index dae50fcc860..ff393682f9b 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -208,7 +208,7 @@ public: std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - _tuneFileDocumentDB); + _tuneFileDocumentDB, _hwInfo); _configMgr.forwardConfig(b); _configMgr.nextGeneration(0); if (! FastOS_File::MakeDirectory((std::string("tmpdb/") + docTypeName).c_str())) { abort(); } diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index e655ae0d739..e39fd1fe28a 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -284,7 +284,7 @@ struct MyConfigSnapshot std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - tuneFileDocumentDB); + tuneFileDocumentDB, HwInfo()); config::DirSpec spec(cfgDir); DocumentDBConfigHelper mgr(spec, "searchdocument"); mgr.forwardConfig(_bootstrap); diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index f93930d189b..9f6eb0ea4e1 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -99,7 +99,7 @@ Fixture::Fixture() std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - tuneFileDocumentDB)); + tuneFileDocumentDB, HwInfo())); mgr.forwardConfig(b); mgr.nextGeneration(0); _db.reset(new DocumentDB(".", mgr.getConfig(), "tcp/localhost:9014", _queryLimiter, _clock, DocTypeName("typea"), diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index 6b1031fe558..dd35d823b99 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -40,13 +40,12 @@ makeBaseConfigSnapshot() DBCM dbcm(spec, "test"); DocumenttypesConfigSP dtcfg(config::ConfigGetter<DocumenttypesConfig>::getConfig("", spec).release()); - BootstrapConfig::SP b(new BootstrapConfig(1, - dtcfg, + BootstrapConfig::SP b(new BootstrapConfig(1, dtcfg, DocumentTypeRepo::SP(new DocumentTypeRepo(*dtcfg)), std::make_shared<ProtonConfig>(), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(), - std::make_shared<TuneFileDocumentDB>())); + std::make_shared<TuneFileDocumentDB>(), HwInfo())); dbcm.forwardConfig(b); dbcm.nextGeneration(0); DocumentDBConfig::SP snap = dbcm.getConfig(); diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 37e4dfc486d..96f742a172c 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -14,12 +14,12 @@ #include <vespa/searchcore/proton/server/proton_config_fetcher.h> #include <vespa/searchcore/proton/server/proton_config_snapshot.h> #include <vespa/searchcore/proton/server/i_proton_configurer.h> +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/varholder.h> #include <vespa/config-bucketspaces.h> -#include <mutex> using namespace config; using namespace proton; @@ -146,14 +146,15 @@ struct ConfigTestFixture { documenttypesBuilder == bootstrapConfig->getDocumenttypesConfig()); } - BootstrapConfig::SP getBootstrapConfig(int64_t generation) const { + BootstrapConfig::SP getBootstrapConfig(int64_t generation, const HwInfo & hwInfo) const { return BootstrapConfig::SP(new BootstrapConfig(generation, - BootstrapConfig::DocumenttypesConfigSP(new DocumenttypesConfig(documenttypesBuilder)), - DocumentTypeRepo::SP(new DocumentTypeRepo(documenttypesBuilder)), - BootstrapConfig::ProtonConfigSP(new ProtonConfig(protonBuilder)), + std::make_shared<DocumenttypesConfig>(documenttypesBuilder), + std::make_shared<DocumentTypeRepo>(documenttypesBuilder), + std::make_shared<ProtonConfig>(protonBuilder), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(bucketspacesBuilder), - std::make_shared<TuneFileDocumentDB>())); + std::make_shared<TuneFileDocumentDB>(), + hwInfo)); } void reload() { context->reload(); } @@ -232,14 +233,20 @@ TEST_FFF("require that bootstrap config manager updates config", ConfigTestFixtu } DocumentDBConfig::SP -getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr) +getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr, const HwInfo & hwInfo) { ConfigRetriever retriever(mgr.createConfigKeySet(), f.context); - mgr.forwardConfig(f.getBootstrapConfig(1)); + mgr.forwardConfig(f.getBootstrapConfig(1, hwInfo)); mgr.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs return mgr.getConfig(); } +DocumentDBConfig::SP +getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr) +{ + return getDocumentDBConfig(f, mgr, HwInfo()); +} + TEST_FF("require that documentdb config manager subscribes for config", ConfigTestFixture("search"), DocumentDBConfigManager(f1.configId + "/typea", "typea")) { @@ -351,4 +358,38 @@ TEST_FF("require that prune removed documents interval can be set based on age", EXPECT_EQUAL(20, config->getMaintenanceConfigSP()->getPruneRemovedDocumentsConfig().getInterval()); } +TEST_FF("require that docstore config computes cachesize automatically if unset", + ConfigTestFixture("test"), + DocumentDBConfigManager(f1.configId + "/test", "test")) +{ + HwInfo hwInfo(HwInfo::Disk(1, false, false), HwInfo::Memory(1000000), HwInfo::Cpu(1)); + f1.addDocType("test"); + f1.protonBuilder.summary.cache.maxbytes = 2000; + auto config = getDocumentDBConfig(f1, f2, hwInfo); + EXPECT_EQUAL(2000ul, config->getStoreConfig().getMaxCacheBytes()); + + f1.protonBuilder.summary.cache.maxbytes = -7; + config = getDocumentDBConfig(f1, f2, hwInfo); + EXPECT_EQUAL(70000ul, config->getStoreConfig().getMaxCacheBytes()); + + f1.protonBuilder.summary.cache.maxbytes = -700; + config = getDocumentDBConfig(f1, f2, hwInfo); + EXPECT_EQUAL(500000ul, config->getStoreConfig().getMaxCacheBytes()); +} + +TEST("test HwInfo equality") { + EXPECT_TRUE(HwInfo::Cpu(1) == HwInfo::Cpu(1)); + EXPECT_FALSE(HwInfo::Cpu(1) == HwInfo::Cpu(2)); + EXPECT_TRUE(HwInfo::Memory(1) == HwInfo::Memory(1)); + EXPECT_FALSE(HwInfo::Memory(1) == HwInfo::Memory(2)); + EXPECT_TRUE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, false,false)); + EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, false,true)); + EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, true,false)); + EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(2, false,false)); + EXPECT_TRUE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 1ul, 1ul)); + EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 1ul, 2ul)); + EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 2ul, 1ul)); + EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(2, false,false), 1ul, 1ul)); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index a8336e29f9f..d59a67d73bb 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -171,7 +171,7 @@ struct ConfigFixture { BootstrapConfig::ProtonConfigSP(new ProtonConfig(_protonBuilder)), std::make_shared<FiledistributorrpcConfig>(), std::make_shared<BucketspacesConfig>(_bucketspacesBuilder), - std::make_shared<TuneFileDocumentDB>())); + std::make_shared<TuneFileDocumentDB>(), HwInfo())); } std::shared_ptr<ProtonConfigSnapshot> getConfigSnapshot() diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index d7b1cbfa45f..c7a3fc7cb62 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -2,7 +2,7 @@ namespace=vespa.config.search.core ## Base directory. The default is ignored as it is assigned by the model -basedir string default="tmp" restart +basedir string default="." restart ## specifies the port number for the persistent internal transport ## protocol provided for a multi-level dispatch system. @@ -210,7 +210,9 @@ grow.numdocs int default=10000 restart grow.multivalueallocfactor double default=0.2 restart ## Control cache size in bytes. -summary.cache.maxbytes long default=0 +## Postive numbers are absolute in bytes. +## Negative numbers are a percentage of memory. +summary.cache.maxbytes long default=-5 ## Include visits in the cache, if the visitoperation allows it. ## This will enable another separate cache of summary.cache.maxbytes size. diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info.h b/searchcore/src/vespa/searchcore/proton/common/hw_info.h index cd0ef2817d7..9129aa43483 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info.h +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info.h @@ -23,6 +23,9 @@ public: uint64_t sizeBytes() const { return _sizeBytes; } bool slow() const { return _slow; } bool shared() const { return _shared; } + bool operator == (const Disk & rhs) const { + return (_sizeBytes == rhs._sizeBytes) && (_slow == rhs._slow) && (_shared == rhs._shared); + } }; class Memory { @@ -31,6 +34,7 @@ public: public: Memory(uint64_t sizeBytes_) : _sizeBytes(sizeBytes_) {} uint64_t sizeBytes() const { return _sizeBytes; } + bool operator == (const Memory & rhs) const { return _sizeBytes == rhs._sizeBytes; } }; class Cpu { @@ -39,6 +43,7 @@ public: public: Cpu(uint32_t cores_) : _cores(cores_) {} uint32_t cores() const { return _cores; } + bool operator == (const Cpu & rhs) const { return _cores == rhs._cores; } }; private: @@ -66,6 +71,9 @@ public: const Disk &disk() const { return _disk; } const Memory &memory() const { return _memory; } const Cpu &cpu() const { return _cpu; } + bool operator == (const HwInfo & rhs) const { + return (_cpu == rhs._cpu) && (_disk == rhs._disk) && (_memory == rhs._memory); + } }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp index 27d430b2893..1bc5cf00ad0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp @@ -29,29 +29,30 @@ BootstrapConfig::BootstrapConfig( const ProtonConfigSP &protonConfig, const FiledistributorrpcConfigSP &filedistRpcConfSP, const BucketspacesConfigSP &bucketspaces, - const search::TuneFileDocumentDB::SP &tuneFileDocumentDB) + const search::TuneFileDocumentDB::SP &tuneFileDocumentDB, + const HwInfo & hwInfo) : _documenttypes(documenttypes), _repo(repo), _proton(protonConfig), _fileDistributorRpc(filedistRpcConfSP), _bucketspaces(bucketspaces), _tuneFileDocumentDB(tuneFileDocumentDB), + _hwInfo(hwInfo), _generation(generation) { } -BootstrapConfig::~BootstrapConfig() { } +BootstrapConfig::~BootstrapConfig() = default; bool BootstrapConfig::operator==(const BootstrapConfig &rhs) const { - return equals<DocumenttypesConfig>(_documenttypes.get(), - rhs._documenttypes.get()) && + return equals<DocumenttypesConfig>(_documenttypes.get(), rhs._documenttypes.get()) && _repo.get() == rhs._repo.get() && equals<ProtonConfig>(_proton.get(), rhs._proton.get()) && equals<FiledistributorrpcConfig>(_fileDistributorRpc.get(), rhs._fileDistributorRpc.get()) && equals<BucketspacesConfig>(_bucketspaces.get(), rhs._bucketspaces.get()) && - equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(), - rhs._tuneFileDocumentDB.get()); + equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(), rhs._tuneFileDocumentDB.get()) && + (_hwInfo == rhs._hwInfo); } @@ -61,5 +62,4 @@ BootstrapConfig::valid() const return _documenttypes && _repo && _proton && _fileDistributorRpc && _bucketspaces && _tuneFileDocumentDB; } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h index 1f888ac1739..368cc30d8a4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h @@ -3,11 +3,11 @@ #pragma once #include "documentdbconfig.h" +#include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/config/config-proton.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> -#include <vespa/searchcore/config/config-proton.h> #include <vespa/searchlib/common/tunefileinfo.h> -#include <vespa/config/retriever/configkeyset.h> #include <vespa/config/retriever/configsnapshot.h> #include <vespa/fileacquirer/config-filedistributorrpc.h> @@ -37,6 +37,7 @@ private: FiledistributorrpcConfigSP _fileDistributorRpc; BucketspacesConfigSP _bucketspaces; search::TuneFileDocumentDB::SP _tuneFileDocumentDB; + HwInfo _hwInfo; int64_t _generation; public: @@ -46,37 +47,21 @@ public: const ProtonConfigSP &protonConfig, const FiledistributorrpcConfigSP &filedistRpcConfSP, const BucketspacesConfigSP &bucketspaces, - const search::TuneFileDocumentDB::SP & - _tuneFileDocumentDB); + const search::TuneFileDocumentDB::SP &_tuneFileDocumentDB, + const HwInfo & hwInfo); ~BootstrapConfig(); - const document::DocumenttypesConfig & - getDocumenttypesConfig() const { return *_documenttypes; } - - const cloud::config::filedistribution::FiledistributorrpcConfig & - getFiledistributorrpcConfig() const { return *_fileDistributorRpc; } - - const FiledistributorrpcConfigSP & - getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; } - - const DocumenttypesConfigSP & - getDocumenttypesConfigSP() const { return _documenttypes; } - - const document::DocumentTypeRepo::SP & - getDocumentTypeRepoSP() const { return _repo; } - - const vespa::config::search::core::ProtonConfig & - getProtonConfig() const { return *_proton; } - - const ProtonConfigSP & - getProtonConfigSP() const { return _proton; } - + const document::DocumenttypesConfig &getDocumenttypesConfig() const { return *_documenttypes; } + const FiledistributorrpcConfig &getFiledistributorrpcConfig() const { return *_fileDistributorRpc; } + const FiledistributorrpcConfigSP &getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; } + const DocumenttypesConfigSP &getDocumenttypesConfigSP() const { return _documenttypes; } + const document::DocumentTypeRepo::SP &getDocumentTypeRepoSP() const { return _repo; } + const vespa::config::search::core::ProtonConfig &getProtonConfig() const { return *_proton; } + const ProtonConfigSP &getProtonConfigSP() const { return _proton; } const BucketspacesConfigSP &getBucketspacesConfigSP() const { return _bucketspaces; } - - const search::TuneFileDocumentDB::SP & - getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; } - + const search::TuneFileDocumentDB::SP &getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; } int64_t getGeneration() const { return _generation; } + const HwInfo & getHwInfo() const { return _hwInfo; } /** * Shared pointers are checked for identity, not equality. @@ -86,4 +71,3 @@ public: }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp index 9088a0b3b88..5459d60e769 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp @@ -2,6 +2,7 @@ #include "bootstrapconfigmanager.h" #include "bootstrapconfig.h" +#include <vespa/searchcore/proton/common/hw_info_sampler.h> #include <vespa/config-bucketspaces.h> #include <vespa/searchlib/common/tunefileinfo.hpp> @@ -38,6 +39,13 @@ BootstrapConfigManager::createConfigKeySet() const .add<BucketspacesConfig>(_configId); } +std::shared_ptr<BootstrapConfig> +BootstrapConfigManager::getConfig() const +{ + std::lock_guard<std::mutex> lock(_pendingConfigMutex); + return _pendingConfigSnapshot; +} + void BootstrapConfigManager::update(const ConfigSnapshot & snapshot) { @@ -83,21 +91,17 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) if (snapshot.isChanged<FiledistributorrpcConfig>(_configId, currentGen)) { LOG(info, "Filedistributorrpc config is changed"); - auto p = snapshot.getConfig<FiledistributorrpcConfig>(_configId); - newFiledistRpcConfSP = BootstrapConfig::FiledistributorrpcConfigSP(std::move(p)); + newFiledistRpcConfSP = snapshot.getConfig<FiledistributorrpcConfig>(_configId); } if (snapshot.isChanged<DocumenttypesConfig>(_configId, currentGen)) { LOG(spam, "Documenttypes config is changed"); - std::unique_ptr<DocumenttypesConfig> documenttypesConfig = snapshot.getConfig<DocumenttypesConfig>(_configId); - DocumentTypeRepo::SP repo(new DocumentTypeRepo(*documenttypesConfig)); - newDocumenttypesConfig = DocumenttypesConfigSP(documenttypesConfig.release()); - newRepo = repo; + newDocumenttypesConfig = snapshot.getConfig<DocumenttypesConfig>(_configId); + newRepo = std::make_shared<DocumentTypeRepo>(*newDocumenttypesConfig); } if (snapshot.isChanged<BucketspacesConfig>(_configId, currentGen)) { LOG(spam, "Bucketspaces config is changed"); - std::unique_ptr<BucketspacesConfig> bucketspacesConfig = snapshot.getConfig<BucketspacesConfig>(_configId); - newBucketspacesConfig = std::move(bucketspacesConfig); + newBucketspacesConfig = snapshot.getConfig<BucketspacesConfig>(_configId); } assert(newProtonConfig); assert(newFiledistRpcConfSP); @@ -106,8 +110,17 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) assert(newDocumenttypesConfig); assert(newRepo); + const ProtonConfig &protonConfig = *newProtonConfig; + const auto &hwDiskCfg = protonConfig.hwinfo.disk; + const auto &hwMemoryCfg = protonConfig.hwinfo.memory; + const auto &hwCpuCfg = protonConfig.hwinfo.cpu; + HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit, + hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores); + HwInfoSampler sampler(protonConfig.basedir, samplerCfg); + auto newSnapshot(std::make_shared<BootstrapConfig>(snapshot.getGeneration(), newDocumenttypesConfig, newRepo, - newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig, newTuneFileDocumentDB)); + newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig, + newTuneFileDocumentDB, sampler.hwInfo())); assert(newSnapshot->valid()); { diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h index 3862f11cdd9..0b0dd341975 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h @@ -20,12 +20,7 @@ public: ~BootstrapConfigManager(); const config::ConfigKeySet createConfigKeySet() const; - std::shared_ptr<BootstrapConfig> - getConfig() const - { - std::lock_guard<std::mutex> lock(_pendingConfigMutex); - return _pendingConfigSnapshot; - } + std::shared_ptr<BootstrapConfig> getConfig() const; void update(const config::ConfigSnapshot & snapshot); private: diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index def18b76360..ec22d3293c4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -2,6 +2,8 @@ #include "documentdbconfigmanager.h" #include "bootstrapconfig.h" +#include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/config-imported-fields.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-summarymap.h> @@ -10,7 +12,6 @@ #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchsummary/config/config-juniperrc.h> -#include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/time/time_box.h> #include <vespa/log/log.h> @@ -157,14 +158,17 @@ deriveCompression(const T & config) { } DocumentStore::Config -getStoreConfig(const ProtonConfig::Summary::Cache & cache) +getStoreConfig(const ProtonConfig::Summary::Cache & cache, const HwInfo & hwInfo) { - return DocumentStore::Config(deriveCompression(cache.compression), cache.maxbytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching); + size_t maxBytes = (cache.maxbytes < 0) + ? (hwInfo.memory().sizeBytes()*std::min(50l, -cache.maxbytes))/100l + : cache.maxbytes; + return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching); } LogDocumentStore::Config -deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush) { - DocumentStore::Config config(getStoreConfig(summary.cache)); +deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush, const HwInfo & hwInfo) { + DocumentStore::Config config(getStoreConfig(summary.cache, hwInfo)); const ProtonConfig::Summary::Log & log(summary.log); const ProtonConfig::Summary::Log::Chunk & chunk(log.chunk); WriteableFileChunk::Config fileConfig(deriveCompression(chunk.compression), chunk.maxbytes); @@ -177,8 +181,8 @@ deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::M return LogDocumentStore::Config(config, logConfig); } -search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton) { - return deriveConfig(proton.summary, proton.flush.memory); +search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton, const HwInfo & hwInfo) { + return deriveConfig(proton.summary, proton.flush.memory, hwInfo); } using AttributesConfigSP = DocumentDBConfig::AttributesConfigSP; @@ -255,10 +259,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) } if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration)) { - newRankProfilesConfig = - RankProfilesConfigSP( - snapshot.getConfig<RankProfilesConfig>(_configId). - release()); + newRankProfilesConfig = snapshot.getConfig<RankProfilesConfig>(_configId); } if (snapshot.isChanged<RankingConstantsConfig>(_configId, currentGeneration)) { RankingConstantsConfigSP newRankingConstantsConfig = RankingConstantsConfigSP( @@ -286,34 +287,34 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) newRankingConstants = std::make_shared<RankingConstants>(constants); } if (snapshot.isChanged<IndexschemaConfig>(_configId, currentGeneration)) { - std::unique_ptr<IndexschemaConfig> indexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId); + newIndexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId); search::index::Schema schema; - search::index::SchemaBuilder::build(*indexschemaConfig, schema); + search::index::SchemaBuilder::build(*newIndexschemaConfig, schema); if (!search::index::SchemaUtil::validateSchema(schema)) { LOG(error, "Cannot use bad index schema, validation failed"); abort(); } - newIndexschemaConfig = IndexschemaConfigSP(indexschemaConfig.release()); } if (snapshot.isChanged<AttributesConfig>(_configId, currentGeneration)) { - newAttributesConfig = AttributesConfigSP(snapshot.getConfig<AttributesConfig>(_configId).release()); + newAttributesConfig = snapshot.getConfig<AttributesConfig>(_configId); } if (snapshot.isChanged<SummaryConfig>(_configId, currentGeneration)) { - newSummaryConfig = SummaryConfigSP(snapshot.getConfig<SummaryConfig>(_configId).release()); + newSummaryConfig = snapshot.getConfig<SummaryConfig>(_configId); } if (snapshot.isChanged<SummarymapConfig>(_configId, currentGeneration)) { - newSummarymapConfig = SummarymapConfigSP(snapshot.getConfig<SummarymapConfig>(_configId).release()); + newSummarymapConfig = snapshot.getConfig<SummarymapConfig>(_configId); } if (snapshot.isChanged<JuniperrcConfig>(_configId, currentGeneration)) { - newJuniperrcConfig = JuniperrcConfigSP(snapshot.getConfig<JuniperrcConfig>(_configId).release()); + newJuniperrcConfig = snapshot.getConfig<JuniperrcConfig>(_configId); } if (snapshot.isChanged<ImportedFieldsConfig>(_configId, currentGeneration)) { - newImportedFieldsConfig = ImportedFieldsConfigSP(snapshot.getConfig<ImportedFieldsConfig>(_configId).release()); + newImportedFieldsConfig = snapshot.getConfig<ImportedFieldsConfig>(_configId); } Schema::SP schema(buildSchema(*newAttributesConfig, *newSummaryConfig, *newIndexschemaConfig)); newMaintenanceConfig = buildMaintenanceConfig(_bootstrapConfig, _docTypeName); - search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig()); + search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig(), + _bootstrapConfig->getHwInfo()); if (newMaintenanceConfig && oldMaintenanceConfig && *newMaintenanceConfig == *oldMaintenanceConfig) { newMaintenanceConfig = oldMaintenanceConfig; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h index 80654c7588b..bd1cb66de43 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h @@ -2,14 +2,14 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <vespa/config/config.h> #include "documentdbconfig.h" +#include <vespa/config/config.h> #include <mutex> namespace proton { class BootstrapConfig; + /** * This class manages the subscription for documentdb configs. */ diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index cfb96172800..37a8ee2aadf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -2,6 +2,7 @@ #include "fileconfigmanager.h" #include "bootstrapconfig.h" +#include <vespa/searchcore/proton/common/hw_info_sampler.h> #include <vespa/config/print/fileconfigwriter.h> #include <vespa/config/print/fileconfigsnapshotreader.h> #include <vespa/config/print/fileconfigsnapshotwriter.h> @@ -11,9 +12,7 @@ #include <vespa/config-summarymap.h> #include <vespa/config-rank-profiles.h> #include <vespa/searchsummary/config/config-juniperrc.h> -#include <vespa/fastos/file.h> #include <vespa/config/helper/configgetter.hpp> -#include <fstream> #include <sstream> #include <fcntl.h> @@ -360,8 +359,16 @@ FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, * of default values here instead of the current values from the config * server. */ + const ProtonConfig &protonConfig = *_protonConfig; + const auto &hwDiskCfg = protonConfig.hwinfo.disk; + const auto &hwMemoryCfg = protonConfig.hwinfo.memory; + const auto &hwCpuCfg = protonConfig.hwinfo.cpu; + HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit, + hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores); + HwInfoSampler sampler(protonConfig.basedir, samplerCfg); auto bootstrap = std::make_shared<BootstrapConfig>(1, docTypesCfg, repo, _protonConfig, filedistRpcConf, - bucketspaces,currentSnapshot.getTuneFileDocumentDBSP()); + bucketspaces,currentSnapshot.getTuneFileDocumentDBSP(), + sampler.hwInfo()); dbc.forwardConfig(bootstrap); dbc.nextGeneration(0); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 8ce7e0c79fe..2639958ed70 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -202,8 +202,6 @@ Proton::Proton(const config::ConfigUri & configUri, _initStarted(false), _initComplete(false), _initDocumentDbsInSequence(false), - _hwInfo(), - _hwInfoSampler(), _documentDBReferenceRegistry() { _documentDBReferenceRegistry = std::make_shared<DocumentDBReferenceRegistry>(); @@ -231,22 +229,12 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) { assert( _initStarted && ! _initComplete ); const ProtonConfig &protonConfig = configSnapshot->getProtonConfig(); - const auto &hwDiskCfg = protonConfig.hwinfo.disk; - const auto &hwMemoryCfg = protonConfig.hwinfo.memory; - const auto &hwCpuCfg = protonConfig.hwinfo.cpu; - HwInfoSampler::Config samplerCfg(hwDiskCfg.size, - hwDiskCfg.writespeed, - hwDiskCfg.slowwritespeedlimit, - hwDiskCfg.samplewritesize, - hwDiskCfg.shared, - hwMemoryCfg.size, - hwCpuCfg.cores); - _hwInfoSampler = std::make_unique<HwInfoSampler>(protonConfig.basedir, samplerCfg); - _hwInfo = _hwInfoSampler->hwInfo(); + const HwInfo & hwInfo = configSnapshot->getHwInfo(); + setFS4Compression(protonConfig); _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler> (protonConfig.basedir, - diskMemUsageSamplerConfig(protonConfig, _hwInfo)); + diskMemUsageSamplerConfig(protonConfig, hwInfo)); _metricsEngine.reset(new MetricsEngine()); _metricsEngine->addMetricsHook(_metricsHook); @@ -263,8 +251,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) switch (flush.strategy) { case ProtonConfig::Flush::MEMORY: { auto memoryFlush = std::make_shared<MemoryFlush>( - MemoryFlushConfigUpdater::convertConfig(flush.memory, _hwInfo.memory()), fastos::ClockSystem::now()); - _memoryFlushConfigUpdater = std::make_unique<MemoryFlushConfigUpdater>(memoryFlush, flush.memory, _hwInfo.memory()); + MemoryFlushConfigUpdater::convertConfig(flush.memory, hwInfo.memory()), fastos::ClockSystem::now()); + _memoryFlushConfigUpdater = std::make_unique<MemoryFlushConfigUpdater>(memoryFlush, flush.memory, hwInfo.memory()); _diskMemUsageSampler->notifier().addDiskMemUsageListener(_memoryFlushConfigUpdater.get()); strategy = memoryFlush; break; @@ -296,7 +284,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::string fileConfigId; _warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024)); - const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, _hwInfo.cpu()); + const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu()); _summaryExecutor.reset(new vespalib::BlockingThreadStackExecutor(summaryThreads, 128*1024, summaryThreads*16)); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { @@ -351,7 +339,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot) protonConfig.search.memory.limiter.minhits); const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP(); - _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, _hwInfo)); + _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, configSnapshot->getHwInfo())); if (_memoryFlushConfigUpdater) { _memoryFlushConfigUpdater->setConfig(protonConfig.flush.memory); _flushEngine->kick(); @@ -545,7 +533,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, _fileHeaderContext, std::move(config_store), initializeThreads, - _hwInfo)); + bootstrapConfig->getHwInfo())); try { ret->start(); } catch (vespalib::Exception &e) { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 8d1026340ca..501b5ab8c62 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -12,7 +12,6 @@ #include "proton_configurer.h" #include "rpc_hooks.h" #include "bootstrapconfig.h" -#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/flushengine/flushengine.h> #include <vespa/searchcore/proton/matchengine/matchengine.h> #include <vespa/searchcore/proton/matching/querylimiter.h> @@ -38,7 +37,6 @@ namespace proton { class DiskMemUsageSampler; -class HwInfoSampler; class IDocumentDBReferenceRegistry; class Proton : public IProtonConfigurerOwner, @@ -124,8 +122,6 @@ private: bool _initStarted; bool _initComplete; bool _initDocumentDbsInSequence; - HwInfo _hwInfo; - std::unique_ptr<HwInfoSampler> _hwInfoSampler; std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry; IDocumentDBConfigOwner * diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp index deeec695f26..f2deb456090 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp @@ -65,8 +65,7 @@ ProtonConfigFetcher::pruneManagerMap(const BootstrapConfig::SP & config) if (_dbManagerMap.find(docTypeName) != _dbManagerMap.end()) { mgr = _dbManagerMap[docTypeName]; } else { - mgr = DocumentDBConfigManager::SP(new DocumentDBConfigManager - (ddb.configid, docTypeName.getName())); + mgr = std::make_shared<DocumentDBConfigManager>(ddb.configid, docTypeName.getName()); } set.add(mgr->createConfigKeySet()); newMap[docTypeName] = mgr; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h index c8d1e55e4e4..8ba56deb7e8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h @@ -52,9 +52,9 @@ private: using TimePoint = std::chrono::time_point<Clock>; using OldDocumentTypeRepo = std::pair<TimePoint, std::shared_ptr<document::DocumentTypeRepo>>; - BootstrapConfigManager _bootstrapConfigManager; + BootstrapConfigManager _bootstrapConfigManager; config::ConfigRetriever _retriever; - IProtonConfigurer & _owner; + IProtonConfigurer & _owner; mutable std::mutex _mutex; // Protects maps using lock_guard = std::lock_guard<std::mutex>; @@ -71,6 +71,4 @@ private: void rememberDocumentTypeRepo(std::shared_ptr<document::DocumentTypeRepo> repo); }; - } // namespace proton - |