diff options
author | Henning Baldersheim <balder@oath.com> | 2018-01-30 12:12:25 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@oath.com> | 2018-01-30 12:12:25 +0000 |
commit | 700bd36c212986332d14674c7c7e32af994b097c (patch) | |
tree | 4e3bbc91d4e0e57c44fb5639965fb5267e60e428 /searchcore | |
parent | 08b55bd6f23e3dccf896febf8de24db273f2b0e8 (diff) |
Let cache be computed based on available memory.
Diffstat (limited to 'searchcore')
8 files changed, 57 insertions, 28 deletions
diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 37e4dfc486d..d305cbf532b 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -14,12 +14,12 @@ #include <vespa/searchcore/proton/server/proton_config_fetcher.h> #include <vespa/searchcore/proton/server/proton_config_snapshot.h> #include <vespa/searchcore/proton/server/i_proton_configurer.h> +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/varholder.h> #include <vespa/config-bucketspaces.h> -#include <mutex> using namespace config; using namespace proton; @@ -60,6 +60,7 @@ struct ConfigTestFixture { ConfigSet set; IConfigContext::SP context; int idcounter; + HwInfo hwInfo; ConfigTestFixture(const std::string & id) : configId(id), @@ -232,14 +233,20 @@ TEST_FFF("require that bootstrap config manager updates config", ConfigTestFixtu } DocumentDBConfig::SP -getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr) +getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr, const HwInfo & hwInfo) { ConfigRetriever retriever(mgr.createConfigKeySet(), f.context); mgr.forwardConfig(f.getBootstrapConfig(1)); - mgr.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs + mgr.update(retriever.getBootstrapConfigs(), hwInfo); // Cheating, but we only need the configs return mgr.getConfig(); } +DocumentDBConfig::SP +getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr) +{ + return getDocumentDBConfig(f, mgr, HwInfo()); +} + TEST_FF("require that documentdb config manager subscribes for config", ConfigTestFixture("search"), DocumentDBConfigManager(f1.configId + "/typea", "typea")) { @@ -274,7 +281,7 @@ TEST_FF("require that documentdb config manager builds schema with imported attr TEST_FFF("require that proton config fetcher follows changes to bootstrap", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { f3.start(); ASSERT_TRUE(f2._configured); ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig())); @@ -289,7 +296,7 @@ TEST_FFF("require that proton config fetcher follows changes to bootstrap", TEST_FFF("require that proton config fetcher follows changes to doctypes", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { f3.start(); f2._configured = false; @@ -309,7 +316,7 @@ TEST_FFF("require that proton config fetcher follows changes to doctypes", TEST_FFF("require that proton config fetcher reconfigures dbowners", ConfigTestFixture("search"), ProtonConfigOwner(), - ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f2, 60000)) { + ProtonConfigFetcher(ConfigUri(f1.configId, f1.context), f1.hwInfo, f2, 60000)) { f3.start(); ASSERT_FALSE(f2.getDocumentDBConfig("typea")); @@ -351,4 +358,18 @@ TEST_FF("require that prune removed documents interval can be set based on age", EXPECT_EQUAL(20, config->getMaintenanceConfigSP()->getPruneRemovedDocumentsConfig().getInterval()); } +TEST_FF("require that docstore config computes cachesize automatically if unset", + ConfigTestFixture("test"), + DocumentDBConfigManager(f1.configId + "/test", "test")) +{ + HwInfo hwInfo(HwInfo::Disk(1, false, false), HwInfo::Memory(1000000), HwInfo::Cpu(1)); + f1.addDocType("test"); + f1.protonBuilder.summary.cache.maxbytes = 2000; + auto config = getDocumentDBConfig(f1, f2, hwInfo); + EXPECT_EQUAL(2000ul, config->getStoreConfig().getMaxCacheBytes()); + f1.protonBuilder.summary.cache.maxbytes = -1; + config = getDocumentDBConfig(f1, f2, hwInfo); + EXPECT_EQUAL(50000ul, config->getStoreConfig().getMaxCacheBytes()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h index 1f888ac1739..1e2c269178f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h @@ -46,8 +46,7 @@ public: const ProtonConfigSP &protonConfig, const FiledistributorrpcConfigSP &filedistRpcConfSP, const BucketspacesConfigSP &bucketspaces, - const search::TuneFileDocumentDB::SP & - _tuneFileDocumentDB); + const search::TuneFileDocumentDB::SP &_tuneFileDocumentDB); ~BootstrapConfig(); const document::DocumenttypesConfig & diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index def18b76360..27bbd019ff5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -2,6 +2,8 @@ #include "documentdbconfigmanager.h" #include "bootstrapconfig.h" +#include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/config-imported-fields.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-summarymap.h> @@ -10,7 +12,6 @@ #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchsummary/config/config-juniperrc.h> -#include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/time/time_box.h> #include <vespa/log/log.h> @@ -157,14 +158,15 @@ deriveCompression(const T & config) { } DocumentStore::Config -getStoreConfig(const ProtonConfig::Summary::Cache & cache) +getStoreConfig(const ProtonConfig::Summary::Cache & cache, const HwInfo & hwInfo) { - return DocumentStore::Config(deriveCompression(cache.compression), cache.maxbytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching); + size_t maxBytes = (cache.maxbytes < 0) ? hwInfo.memory().sizeBytes()*0.05 : cache.maxbytes; + return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching); } LogDocumentStore::Config -deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush) { - DocumentStore::Config config(getStoreConfig(summary.cache)); +deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush, const HwInfo & hwInfo) { + DocumentStore::Config config(getStoreConfig(summary.cache, hwInfo)); const ProtonConfig::Summary::Log & log(summary.log); const ProtonConfig::Summary::Log::Chunk & chunk(log.chunk); WriteableFileChunk::Config fileConfig(deriveCompression(chunk.compression), chunk.maxbytes); @@ -177,8 +179,8 @@ deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::M return LogDocumentStore::Config(config, logConfig); } -search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton) { - return deriveConfig(proton.summary, proton.flush.memory); +search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton, const HwInfo & hwInfo) { + return deriveConfig(proton.summary, proton.flush.memory, hwInfo); } using AttributesConfigSP = DocumentDBConfig::AttributesConfigSP; @@ -201,7 +203,7 @@ filterImportedAttributes(const AttributesConfigSP &attrCfg) } void -DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) +DocumentDBConfigManager::update(const ConfigSnapshot &snapshot, const HwInfo & hwInfo) { using RankProfilesConfigSP = DocumentDBConfig::RankProfilesConfigSP; using RankingConstantsConfigSP = std::shared_ptr<vespa::config::search::core::RankingConstantsConfig>; @@ -313,7 +315,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) Schema::SP schema(buildSchema(*newAttributesConfig, *newSummaryConfig, *newIndexschemaConfig)); newMaintenanceConfig = buildMaintenanceConfig(_bootstrapConfig, _docTypeName); - search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig()); + search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig(), hwInfo); if (newMaintenanceConfig && oldMaintenanceConfig && *newMaintenanceConfig == *oldMaintenanceConfig) { newMaintenanceConfig = oldMaintenanceConfig; } @@ -375,7 +377,8 @@ forwardConfig(const BootstrapConfig::SP & config) } DocumentDBConfigHelper::DocumentDBConfigHelper(const DirSpec &spec, const vespalib::string &docTypeName) - : _mgr("", docTypeName), + : _hwInfo(std::make_unique<HwInfo>()), + _mgr("", docTypeName), _retriever(make_unique<ConfigRetriever>(_mgr.createConfigKeySet(), make_shared<ConfigContext>(spec))) { } @@ -387,7 +390,7 @@ DocumentDBConfigHelper::nextGeneration(int timeoutInMillis) ConfigSnapshot snapshot(_retriever->getBootstrapConfigs(timeoutInMillis)); if (snapshot.empty()) return false; - _mgr.update(snapshot); + _mgr.update(snapshot, *_hwInfo); return true; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h index 80654c7588b..44e0c111f47 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h @@ -10,6 +10,8 @@ namespace proton { class BootstrapConfig; +class HwInfo; + /** * This class manages the subscription for documentdb configs. */ @@ -35,7 +37,7 @@ private: public: DocumentDBConfigManager(const vespalib::string &configId, const vespalib::string &docTypeName); ~DocumentDBConfigManager(); - void update(const config::ConfigSnapshot & snapshot); + void update(const config::ConfigSnapshot & snapshot, const HwInfo & hwInfo); DocumentDBConfig::SP getConfig() const; @@ -57,6 +59,7 @@ public: DocumentDBConfig::SP getConfig() const; void forwardConfig(const std::shared_ptr<BootstrapConfig> & config); private: + std::unique_ptr<HwInfo> _hwInfo; DocumentDBConfigManager _mgr; std::unique_ptr<config::ConfigRetriever> _retriever; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 8ce7e0c79fe..28811fef84a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -188,7 +188,8 @@ Proton::Proton(const config::ConfigUri & configUri, // serializing startup. _executor(1, 128 * 1024), _protonConfigurer(_executor, *this), - _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout), + _hwInfo(), + _protonConfigFetcher(configUri, _hwInfo, _protonConfigurer, subscribeTimeout), _warmupExecutor(), _summaryExecutor(), _queryLimiter(), @@ -202,7 +203,6 @@ Proton::Proton(const config::ConfigUri & configUri, _initStarted(false), _initComplete(false), _initDocumentDbsInSequence(false), - _hwInfo(), _hwInfoSampler(), _documentDBReferenceRegistry() { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 8d1026340ca..e6d7093fd44 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -110,6 +110,7 @@ private: TransportServer::UP _fs4Server; vespalib::ThreadStackExecutor _executor; ProtonConfigurer _protonConfigurer; + HwInfo _hwInfo; ProtonConfigFetcher _protonConfigFetcher; std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor; std::unique_ptr<vespalib::ThreadStackExecutorBase> _summaryExecutor; @@ -124,7 +125,6 @@ private: bool _initStarted; bool _initComplete; bool _initDocumentDbsInSequence; - HwInfo _hwInfo; std::unique_ptr<HwInfoSampler> _hwInfoSampler; std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp index deeec695f26..f83b3f784f3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp @@ -17,10 +17,11 @@ using namespace std::chrono_literals; namespace proton { -ProtonConfigFetcher::ProtonConfigFetcher(const config::ConfigUri & configUri, IProtonConfigurer &owner, uint64_t subscribeTimeout) +ProtonConfigFetcher::ProtonConfigFetcher(const config::ConfigUri & configUri, const HwInfo &hwInfo, IProtonConfigurer &owner, uint64_t subscribeTimeout) : _bootstrapConfigManager(configUri.getConfigId()), _retriever(_bootstrapConfigManager.createConfigKeySet(), configUri.getContext(), subscribeTimeout), _owner(owner), + _hwInfo(hwInfo), _mutex(), _dbManagerMap(), _threadPool(128 * 1024, 1), @@ -81,7 +82,7 @@ ProtonConfigFetcher::updateDocumentDBConfigs(const BootstrapConfig::SP & bootstr lock_guard guard(_mutex); for (auto & entry : _dbManagerMap) { entry.second->forwardConfig(bootstrapConfig); - entry.second->update(snapshot); + entry.second->update(snapshot, _hwInfo); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h index c8d1e55e4e4..7165e364dc5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h @@ -17,6 +17,7 @@ namespace proton { class BootstrapConfig; class IProtonConfigurer; +class HwInfo; /** * A ProtonConfigFetcher monitors all config in proton and document dbs for change @@ -27,7 +28,7 @@ class ProtonConfigFetcher : public FastOS_Runnable public: using BootstrapConfigSP = std::shared_ptr<BootstrapConfig>; - ProtonConfigFetcher(const config::ConfigUri & configUri, IProtonConfigurer &owner, uint64_t subscribeTimeout); + ProtonConfigFetcher(const config::ConfigUri & configUri, const HwInfo & hwInfo,IProtonConfigurer &owner, uint64_t subscribeTimeout); ~ProtonConfigFetcher(); /** * Get the current config generation. @@ -52,9 +53,10 @@ private: using TimePoint = std::chrono::time_point<Clock>; using OldDocumentTypeRepo = std::pair<TimePoint, std::shared_ptr<document::DocumentTypeRepo>>; - BootstrapConfigManager _bootstrapConfigManager; + BootstrapConfigManager _bootstrapConfigManager; config::ConfigRetriever _retriever; - IProtonConfigurer & _owner; + IProtonConfigurer & _owner; + const HwInfo & _hwInfo; mutable std::mutex _mutex; // Protects maps using lock_guard = std::lock_guard<std::mutex>; |