summaryrefslogtreecommitdiffstats
path: root/searchcore
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2018-01-31 18:42:15 +0100
committerGitHub <noreply@github.com>2018-01-31 18:42:15 +0100
commit01dcc51225aed6e017d54875a0c924f0fab7aef1 (patch)
treef538eb6f83f8a0c0d5df4a0c4d383985a92725ab /searchcore
parent2b42b7abba86daa12dbc5691d1873bec14a94d00 (diff)
Revert "Revert "Balder/compute summary cache automatically in backend""
Diffstat (limited to 'searchcore')
-rw-r--r--searchcore/src/apps/tests/persistenceconformance_test.cpp2
-rw-r--r--searchcore/src/tests/proton/docsummary/docsummary.cpp2
-rw-r--r--searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp2
-rw-r--r--searchcore/src/tests/proton/documentdb/documentdb_test.cpp2
-rw-r--r--searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp5
-rw-r--r--searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp57
-rw-r--r--searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/config/proton.def6
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/hw_info.h8
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp14
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h44
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp31
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h7
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp41
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp13
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/proton.cpp28
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/proton.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp3
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h6
20 files changed, 156 insertions, 125 deletions
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp
index 5c59b3376ba..5b50ec49844 100644
--- a/searchcore/src/apps/tests/persistenceconformance_test.cpp
+++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp
@@ -183,7 +183,7 @@ public:
std::make_shared<ProtonConfig>(),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(),
- tuneFileDocDB));
+ tuneFileDocDB, HwInfo()));
mgr.forwardConfig(b);
mgr.nextGeneration(0);
return DocumentDB::SP(
diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp
index dae50fcc860..ff393682f9b 100644
--- a/searchcore/src/tests/proton/docsummary/docsummary.cpp
+++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp
@@ -208,7 +208,7 @@ public:
std::make_shared<ProtonConfig>(),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(),
- _tuneFileDocumentDB);
+ _tuneFileDocumentDB, _hwInfo);
_configMgr.forwardConfig(b);
_configMgr.nextGeneration(0);
if (! FastOS_File::MakeDirectory((std::string("tmpdb/") + docTypeName).c_str())) { abort(); }
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index e655ae0d739..e39fd1fe28a 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -284,7 +284,7 @@ struct MyConfigSnapshot
std::make_shared<ProtonConfig>(),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(),
- tuneFileDocumentDB);
+ tuneFileDocumentDB, HwInfo());
config::DirSpec spec(cfgDir);
DocumentDBConfigHelper mgr(spec, "searchdocument");
mgr.forwardConfig(_bootstrap);
diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
index f93930d189b..9f6eb0ea4e1 100644
--- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
@@ -99,7 +99,7 @@ Fixture::Fixture()
std::make_shared<ProtonConfig>(),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(),
- tuneFileDocumentDB));
+ tuneFileDocumentDB, HwInfo()));
mgr.forwardConfig(b);
mgr.nextGeneration(0);
_db.reset(new DocumentDB(".", mgr.getConfig(), "tcp/localhost:9014", _queryLimiter, _clock, DocTypeName("typea"),
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp
index 6b1031fe558..dd35d823b99 100644
--- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp
@@ -40,13 +40,12 @@ makeBaseConfigSnapshot()
DBCM dbcm(spec, "test");
DocumenttypesConfigSP dtcfg(config::ConfigGetter<DocumenttypesConfig>::getConfig("", spec).release());
- BootstrapConfig::SP b(new BootstrapConfig(1,
- dtcfg,
+ BootstrapConfig::SP b(new BootstrapConfig(1, dtcfg,
DocumentTypeRepo::SP(new DocumentTypeRepo(*dtcfg)),
std::make_shared<ProtonConfig>(),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(),
- std::make_shared<TuneFileDocumentDB>()));
+ std::make_shared<TuneFileDocumentDB>(), HwInfo()));
dbcm.forwardConfig(b);
dbcm.nextGeneration(0);
DocumentDBConfig::SP snap = dbcm.getConfig();
diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
index 37e4dfc486d..96f742a172c 100644
--- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
+++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
@@ -14,12 +14,12 @@
#include <vespa/searchcore/proton/server/proton_config_fetcher.h>
#include <vespa/searchcore/proton/server/proton_config_snapshot.h>
#include <vespa/searchcore/proton/server/i_proton_configurer.h>
+#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchsummary/config/config-juniperrc.h>
#include <vespa/searchcore/config/config-ranking-constants.h>
#include <vespa/vespalib/testkit/testapp.h>
#include <vespa/vespalib/util/varholder.h>
#include <vespa/config-bucketspaces.h>
-#include <mutex>
using namespace config;
using namespace proton;
@@ -146,14 +146,15 @@ struct ConfigTestFixture {
documenttypesBuilder == bootstrapConfig->getDocumenttypesConfig());
}
- BootstrapConfig::SP getBootstrapConfig(int64_t generation) const {
+ BootstrapConfig::SP getBootstrapConfig(int64_t generation, const HwInfo & hwInfo) const {
return BootstrapConfig::SP(new BootstrapConfig(generation,
- BootstrapConfig::DocumenttypesConfigSP(new DocumenttypesConfig(documenttypesBuilder)),
- DocumentTypeRepo::SP(new DocumentTypeRepo(documenttypesBuilder)),
- BootstrapConfig::ProtonConfigSP(new ProtonConfig(protonBuilder)),
+ std::make_shared<DocumenttypesConfig>(documenttypesBuilder),
+ std::make_shared<DocumentTypeRepo>(documenttypesBuilder),
+ std::make_shared<ProtonConfig>(protonBuilder),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(bucketspacesBuilder),
- std::make_shared<TuneFileDocumentDB>()));
+ std::make_shared<TuneFileDocumentDB>(),
+ hwInfo));
}
void reload() { context->reload(); }
@@ -232,14 +233,20 @@ TEST_FFF("require that bootstrap config manager updates config", ConfigTestFixtu
}
DocumentDBConfig::SP
-getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr)
+getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr, const HwInfo & hwInfo)
{
ConfigRetriever retriever(mgr.createConfigKeySet(), f.context);
- mgr.forwardConfig(f.getBootstrapConfig(1));
+ mgr.forwardConfig(f.getBootstrapConfig(1, hwInfo));
mgr.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs
return mgr.getConfig();
}
+DocumentDBConfig::SP
+getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr)
+{
+ return getDocumentDBConfig(f, mgr, HwInfo());
+}
+
TEST_FF("require that documentdb config manager subscribes for config",
ConfigTestFixture("search"),
DocumentDBConfigManager(f1.configId + "/typea", "typea")) {
@@ -351,4 +358,38 @@ TEST_FF("require that prune removed documents interval can be set based on age",
EXPECT_EQUAL(20, config->getMaintenanceConfigSP()->getPruneRemovedDocumentsConfig().getInterval());
}
+TEST_FF("require that docstore config computes cachesize automatically if unset",
+ ConfigTestFixture("test"),
+ DocumentDBConfigManager(f1.configId + "/test", "test"))
+{
+ HwInfo hwInfo(HwInfo::Disk(1, false, false), HwInfo::Memory(1000000), HwInfo::Cpu(1));
+ f1.addDocType("test");
+ f1.protonBuilder.summary.cache.maxbytes = 2000;
+ auto config = getDocumentDBConfig(f1, f2, hwInfo);
+ EXPECT_EQUAL(2000ul, config->getStoreConfig().getMaxCacheBytes());
+
+ f1.protonBuilder.summary.cache.maxbytes = -7;
+ config = getDocumentDBConfig(f1, f2, hwInfo);
+ EXPECT_EQUAL(70000ul, config->getStoreConfig().getMaxCacheBytes());
+
+ f1.protonBuilder.summary.cache.maxbytes = -700;
+ config = getDocumentDBConfig(f1, f2, hwInfo);
+ EXPECT_EQUAL(500000ul, config->getStoreConfig().getMaxCacheBytes());
+}
+
+TEST("test HwInfo equality") {
+ EXPECT_TRUE(HwInfo::Cpu(1) == HwInfo::Cpu(1));
+ EXPECT_FALSE(HwInfo::Cpu(1) == HwInfo::Cpu(2));
+ EXPECT_TRUE(HwInfo::Memory(1) == HwInfo::Memory(1));
+ EXPECT_FALSE(HwInfo::Memory(1) == HwInfo::Memory(2));
+ EXPECT_TRUE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, false,false));
+ EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, false,true));
+ EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, true,false));
+ EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(2, false,false));
+ EXPECT_TRUE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 1ul, 1ul));
+ EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 1ul, 2ul));
+ EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 2ul, 1ul));
+ EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(2, false,false), 1ul, 1ul));
+}
+
TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
index a8336e29f9f..d59a67d73bb 100644
--- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
+++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
@@ -171,7 +171,7 @@ struct ConfigFixture {
BootstrapConfig::ProtonConfigSP(new ProtonConfig(_protonBuilder)),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(_bucketspacesBuilder),
- std::make_shared<TuneFileDocumentDB>()));
+ std::make_shared<TuneFileDocumentDB>(), HwInfo()));
}
std::shared_ptr<ProtonConfigSnapshot> getConfigSnapshot()
diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def
index d7b1cbfa45f..c7a3fc7cb62 100644
--- a/searchcore/src/vespa/searchcore/config/proton.def
+++ b/searchcore/src/vespa/searchcore/config/proton.def
@@ -2,7 +2,7 @@
namespace=vespa.config.search.core
## Base directory. The default is ignored as it is assigned by the model
-basedir string default="tmp" restart
+basedir string default="." restart
## specifies the port number for the persistent internal transport
## protocol provided for a multi-level dispatch system.
@@ -210,7 +210,9 @@ grow.numdocs int default=10000 restart
grow.multivalueallocfactor double default=0.2 restart
## Control cache size in bytes.
-summary.cache.maxbytes long default=0
+## Postive numbers are absolute in bytes.
+## Negative numbers are a percentage of memory.
+summary.cache.maxbytes long default=-5
## Include visits in the cache, if the visitoperation allows it.
## This will enable another separate cache of summary.cache.maxbytes size.
diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info.h b/searchcore/src/vespa/searchcore/proton/common/hw_info.h
index cd0ef2817d7..9129aa43483 100644
--- a/searchcore/src/vespa/searchcore/proton/common/hw_info.h
+++ b/searchcore/src/vespa/searchcore/proton/common/hw_info.h
@@ -23,6 +23,9 @@ public:
uint64_t sizeBytes() const { return _sizeBytes; }
bool slow() const { return _slow; }
bool shared() const { return _shared; }
+ bool operator == (const Disk & rhs) const {
+ return (_sizeBytes == rhs._sizeBytes) && (_slow == rhs._slow) && (_shared == rhs._shared);
+ }
};
class Memory {
@@ -31,6 +34,7 @@ public:
public:
Memory(uint64_t sizeBytes_) : _sizeBytes(sizeBytes_) {}
uint64_t sizeBytes() const { return _sizeBytes; }
+ bool operator == (const Memory & rhs) const { return _sizeBytes == rhs._sizeBytes; }
};
class Cpu {
@@ -39,6 +43,7 @@ public:
public:
Cpu(uint32_t cores_) : _cores(cores_) {}
uint32_t cores() const { return _cores; }
+ bool operator == (const Cpu & rhs) const { return _cores == rhs._cores; }
};
private:
@@ -66,6 +71,9 @@ public:
const Disk &disk() const { return _disk; }
const Memory &memory() const { return _memory; }
const Cpu &cpu() const { return _cpu; }
+ bool operator == (const HwInfo & rhs) const {
+ return (_cpu == rhs._cpu) && (_disk == rhs._disk) && (_memory == rhs._memory);
+ }
};
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp
index 27d430b2893..1bc5cf00ad0 100644
--- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp
@@ -29,29 +29,30 @@ BootstrapConfig::BootstrapConfig(
const ProtonConfigSP &protonConfig,
const FiledistributorrpcConfigSP &filedistRpcConfSP,
const BucketspacesConfigSP &bucketspaces,
- const search::TuneFileDocumentDB::SP &tuneFileDocumentDB)
+ const search::TuneFileDocumentDB::SP &tuneFileDocumentDB,
+ const HwInfo & hwInfo)
: _documenttypes(documenttypes),
_repo(repo),
_proton(protonConfig),
_fileDistributorRpc(filedistRpcConfSP),
_bucketspaces(bucketspaces),
_tuneFileDocumentDB(tuneFileDocumentDB),
+ _hwInfo(hwInfo),
_generation(generation)
{ }
-BootstrapConfig::~BootstrapConfig() { }
+BootstrapConfig::~BootstrapConfig() = default;
bool
BootstrapConfig::operator==(const BootstrapConfig &rhs) const
{
- return equals<DocumenttypesConfig>(_documenttypes.get(),
- rhs._documenttypes.get()) &&
+ return equals<DocumenttypesConfig>(_documenttypes.get(), rhs._documenttypes.get()) &&
_repo.get() == rhs._repo.get() &&
equals<ProtonConfig>(_proton.get(), rhs._proton.get()) &&
equals<FiledistributorrpcConfig>(_fileDistributorRpc.get(), rhs._fileDistributorRpc.get()) &&
equals<BucketspacesConfig>(_bucketspaces.get(), rhs._bucketspaces.get()) &&
- equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(),
- rhs._tuneFileDocumentDB.get());
+ equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(), rhs._tuneFileDocumentDB.get()) &&
+ (_hwInfo == rhs._hwInfo);
}
@@ -61,5 +62,4 @@ BootstrapConfig::valid() const
return _documenttypes && _repo && _proton && _fileDistributorRpc && _bucketspaces && _tuneFileDocumentDB;
}
-
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h
index 1f888ac1739..368cc30d8a4 100644
--- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h
@@ -3,11 +3,11 @@
#pragma once
#include "documentdbconfig.h"
+#include <vespa/searchcore/proton/common/hw_info.h>
+#include <vespa/searchcore/config/config-proton.h>
#include <vespa/document/config/config-documenttypes.h>
#include <vespa/document/repo/documenttyperepo.h>
-#include <vespa/searchcore/config/config-proton.h>
#include <vespa/searchlib/common/tunefileinfo.h>
-#include <vespa/config/retriever/configkeyset.h>
#include <vespa/config/retriever/configsnapshot.h>
#include <vespa/fileacquirer/config-filedistributorrpc.h>
@@ -37,6 +37,7 @@ private:
FiledistributorrpcConfigSP _fileDistributorRpc;
BucketspacesConfigSP _bucketspaces;
search::TuneFileDocumentDB::SP _tuneFileDocumentDB;
+ HwInfo _hwInfo;
int64_t _generation;
public:
@@ -46,37 +47,21 @@ public:
const ProtonConfigSP &protonConfig,
const FiledistributorrpcConfigSP &filedistRpcConfSP,
const BucketspacesConfigSP &bucketspaces,
- const search::TuneFileDocumentDB::SP &
- _tuneFileDocumentDB);
+ const search::TuneFileDocumentDB::SP &_tuneFileDocumentDB,
+ const HwInfo & hwInfo);
~BootstrapConfig();
- const document::DocumenttypesConfig &
- getDocumenttypesConfig() const { return *_documenttypes; }
-
- const cloud::config::filedistribution::FiledistributorrpcConfig &
- getFiledistributorrpcConfig() const { return *_fileDistributorRpc; }
-
- const FiledistributorrpcConfigSP &
- getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; }
-
- const DocumenttypesConfigSP &
- getDocumenttypesConfigSP() const { return _documenttypes; }
-
- const document::DocumentTypeRepo::SP &
- getDocumentTypeRepoSP() const { return _repo; }
-
- const vespa::config::search::core::ProtonConfig &
- getProtonConfig() const { return *_proton; }
-
- const ProtonConfigSP &
- getProtonConfigSP() const { return _proton; }
-
+ const document::DocumenttypesConfig &getDocumenttypesConfig() const { return *_documenttypes; }
+ const FiledistributorrpcConfig &getFiledistributorrpcConfig() const { return *_fileDistributorRpc; }
+ const FiledistributorrpcConfigSP &getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; }
+ const DocumenttypesConfigSP &getDocumenttypesConfigSP() const { return _documenttypes; }
+ const document::DocumentTypeRepo::SP &getDocumentTypeRepoSP() const { return _repo; }
+ const vespa::config::search::core::ProtonConfig &getProtonConfig() const { return *_proton; }
+ const ProtonConfigSP &getProtonConfigSP() const { return _proton; }
const BucketspacesConfigSP &getBucketspacesConfigSP() const { return _bucketspaces; }
-
- const search::TuneFileDocumentDB::SP &
- getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; }
-
+ const search::TuneFileDocumentDB::SP &getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; }
int64_t getGeneration() const { return _generation; }
+ const HwInfo & getHwInfo() const { return _hwInfo; }
/**
* Shared pointers are checked for identity, not equality.
@@ -86,4 +71,3 @@ public:
};
} // namespace proton
-
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp
index 9088a0b3b88..5459d60e769 100644
--- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp
@@ -2,6 +2,7 @@
#include "bootstrapconfigmanager.h"
#include "bootstrapconfig.h"
+#include <vespa/searchcore/proton/common/hw_info_sampler.h>
#include <vespa/config-bucketspaces.h>
#include <vespa/searchlib/common/tunefileinfo.hpp>
@@ -38,6 +39,13 @@ BootstrapConfigManager::createConfigKeySet() const
.add<BucketspacesConfig>(_configId);
}
+std::shared_ptr<BootstrapConfig>
+BootstrapConfigManager::getConfig() const
+{
+ std::lock_guard<std::mutex> lock(_pendingConfigMutex);
+ return _pendingConfigSnapshot;
+}
+
void
BootstrapConfigManager::update(const ConfigSnapshot & snapshot)
{
@@ -83,21 +91,17 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot)
if (snapshot.isChanged<FiledistributorrpcConfig>(_configId, currentGen)) {
LOG(info, "Filedistributorrpc config is changed");
- auto p = snapshot.getConfig<FiledistributorrpcConfig>(_configId);
- newFiledistRpcConfSP = BootstrapConfig::FiledistributorrpcConfigSP(std::move(p));
+ newFiledistRpcConfSP = snapshot.getConfig<FiledistributorrpcConfig>(_configId);
}
if (snapshot.isChanged<DocumenttypesConfig>(_configId, currentGen)) {
LOG(spam, "Documenttypes config is changed");
- std::unique_ptr<DocumenttypesConfig> documenttypesConfig = snapshot.getConfig<DocumenttypesConfig>(_configId);
- DocumentTypeRepo::SP repo(new DocumentTypeRepo(*documenttypesConfig));
- newDocumenttypesConfig = DocumenttypesConfigSP(documenttypesConfig.release());
- newRepo = repo;
+ newDocumenttypesConfig = snapshot.getConfig<DocumenttypesConfig>(_configId);
+ newRepo = std::make_shared<DocumentTypeRepo>(*newDocumenttypesConfig);
}
if (snapshot.isChanged<BucketspacesConfig>(_configId, currentGen)) {
LOG(spam, "Bucketspaces config is changed");
- std::unique_ptr<BucketspacesConfig> bucketspacesConfig = snapshot.getConfig<BucketspacesConfig>(_configId);
- newBucketspacesConfig = std::move(bucketspacesConfig);
+ newBucketspacesConfig = snapshot.getConfig<BucketspacesConfig>(_configId);
}
assert(newProtonConfig);
assert(newFiledistRpcConfSP);
@@ -106,8 +110,17 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot)
assert(newDocumenttypesConfig);
assert(newRepo);
+ const ProtonConfig &protonConfig = *newProtonConfig;
+ const auto &hwDiskCfg = protonConfig.hwinfo.disk;
+ const auto &hwMemoryCfg = protonConfig.hwinfo.memory;
+ const auto &hwCpuCfg = protonConfig.hwinfo.cpu;
+ HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit,
+ hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores);
+ HwInfoSampler sampler(protonConfig.basedir, samplerCfg);
+
auto newSnapshot(std::make_shared<BootstrapConfig>(snapshot.getGeneration(), newDocumenttypesConfig, newRepo,
- newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig, newTuneFileDocumentDB));
+ newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig,
+ newTuneFileDocumentDB, sampler.hwInfo()));
assert(newSnapshot->valid());
{
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h
index 3862f11cdd9..0b0dd341975 100644
--- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h
@@ -20,12 +20,7 @@ public:
~BootstrapConfigManager();
const config::ConfigKeySet createConfigKeySet() const;
- std::shared_ptr<BootstrapConfig>
- getConfig() const
- {
- std::lock_guard<std::mutex> lock(_pendingConfigMutex);
- return _pendingConfigSnapshot;
- }
+ std::shared_ptr<BootstrapConfig> getConfig() const;
void update(const config::ConfigSnapshot & snapshot);
private:
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
index def18b76360..ec22d3293c4 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
@@ -2,6 +2,8 @@
#include "documentdbconfigmanager.h"
#include "bootstrapconfig.h"
+#include <vespa/searchcore/proton/common/hw_info.h>
+#include <vespa/searchcore/config/config-ranking-constants.h>
#include <vespa/config-imported-fields.h>
#include <vespa/config-rank-profiles.h>
#include <vespa/config-summarymap.h>
@@ -10,7 +12,6 @@
#include <vespa/searchcommon/common/schemaconfigurer.h>
#include <vespa/searchlib/index/schemautil.h>
#include <vespa/searchsummary/config/config-juniperrc.h>
-#include <vespa/searchcore/config/config-ranking-constants.h>
#include <vespa/vespalib/time/time_box.h>
#include <vespa/log/log.h>
@@ -157,14 +158,17 @@ deriveCompression(const T & config) {
}
DocumentStore::Config
-getStoreConfig(const ProtonConfig::Summary::Cache & cache)
+getStoreConfig(const ProtonConfig::Summary::Cache & cache, const HwInfo & hwInfo)
{
- return DocumentStore::Config(deriveCompression(cache.compression), cache.maxbytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching);
+ size_t maxBytes = (cache.maxbytes < 0)
+ ? (hwInfo.memory().sizeBytes()*std::min(50l, -cache.maxbytes))/100l
+ : cache.maxbytes;
+ return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching);
}
LogDocumentStore::Config
-deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush) {
- DocumentStore::Config config(getStoreConfig(summary.cache));
+deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush, const HwInfo & hwInfo) {
+ DocumentStore::Config config(getStoreConfig(summary.cache, hwInfo));
const ProtonConfig::Summary::Log & log(summary.log);
const ProtonConfig::Summary::Log::Chunk & chunk(log.chunk);
WriteableFileChunk::Config fileConfig(deriveCompression(chunk.compression), chunk.maxbytes);
@@ -177,8 +181,8 @@ deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::M
return LogDocumentStore::Config(config, logConfig);
}
-search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton) {
- return deriveConfig(proton.summary, proton.flush.memory);
+search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton, const HwInfo & hwInfo) {
+ return deriveConfig(proton.summary, proton.flush.memory, hwInfo);
}
using AttributesConfigSP = DocumentDBConfig::AttributesConfigSP;
@@ -255,10 +259,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot)
}
if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration)) {
- newRankProfilesConfig =
- RankProfilesConfigSP(
- snapshot.getConfig<RankProfilesConfig>(_configId).
- release());
+ newRankProfilesConfig = snapshot.getConfig<RankProfilesConfig>(_configId);
}
if (snapshot.isChanged<RankingConstantsConfig>(_configId, currentGeneration)) {
RankingConstantsConfigSP newRankingConstantsConfig = RankingConstantsConfigSP(
@@ -286,34 +287,34 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot)
newRankingConstants = std::make_shared<RankingConstants>(constants);
}
if (snapshot.isChanged<IndexschemaConfig>(_configId, currentGeneration)) {
- std::unique_ptr<IndexschemaConfig> indexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId);
+ newIndexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId);
search::index::Schema schema;
- search::index::SchemaBuilder::build(*indexschemaConfig, schema);
+ search::index::SchemaBuilder::build(*newIndexschemaConfig, schema);
if (!search::index::SchemaUtil::validateSchema(schema)) {
LOG(error, "Cannot use bad index schema, validation failed");
abort();
}
- newIndexschemaConfig = IndexschemaConfigSP(indexschemaConfig.release());
}
if (snapshot.isChanged<AttributesConfig>(_configId, currentGeneration)) {
- newAttributesConfig = AttributesConfigSP(snapshot.getConfig<AttributesConfig>(_configId).release());
+ newAttributesConfig = snapshot.getConfig<AttributesConfig>(_configId);
}
if (snapshot.isChanged<SummaryConfig>(_configId, currentGeneration)) {
- newSummaryConfig = SummaryConfigSP(snapshot.getConfig<SummaryConfig>(_configId).release());
+ newSummaryConfig = snapshot.getConfig<SummaryConfig>(_configId);
}
if (snapshot.isChanged<SummarymapConfig>(_configId, currentGeneration)) {
- newSummarymapConfig = SummarymapConfigSP(snapshot.getConfig<SummarymapConfig>(_configId).release());
+ newSummarymapConfig = snapshot.getConfig<SummarymapConfig>(_configId);
}
if (snapshot.isChanged<JuniperrcConfig>(_configId, currentGeneration)) {
- newJuniperrcConfig = JuniperrcConfigSP(snapshot.getConfig<JuniperrcConfig>(_configId).release());
+ newJuniperrcConfig = snapshot.getConfig<JuniperrcConfig>(_configId);
}
if (snapshot.isChanged<ImportedFieldsConfig>(_configId, currentGeneration)) {
- newImportedFieldsConfig = ImportedFieldsConfigSP(snapshot.getConfig<ImportedFieldsConfig>(_configId).release());
+ newImportedFieldsConfig = snapshot.getConfig<ImportedFieldsConfig>(_configId);
}
Schema::SP schema(buildSchema(*newAttributesConfig, *newSummaryConfig, *newIndexschemaConfig));
newMaintenanceConfig = buildMaintenanceConfig(_bootstrapConfig, _docTypeName);
- search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig());
+ search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig(),
+ _bootstrapConfig->getHwInfo());
if (newMaintenanceConfig && oldMaintenanceConfig && *newMaintenanceConfig == *oldMaintenanceConfig) {
newMaintenanceConfig = oldMaintenanceConfig;
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h
index 80654c7588b..bd1cb66de43 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h
@@ -2,14 +2,14 @@
#pragma once
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/config/config.h>
#include "documentdbconfig.h"
+#include <vespa/config/config.h>
#include <mutex>
namespace proton {
class BootstrapConfig;
+
/**
* This class manages the subscription for documentdb configs.
*/
diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
index cfb96172800..37a8ee2aadf 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
@@ -2,6 +2,7 @@
#include "fileconfigmanager.h"
#include "bootstrapconfig.h"
+#include <vespa/searchcore/proton/common/hw_info_sampler.h>
#include <vespa/config/print/fileconfigwriter.h>
#include <vespa/config/print/fileconfigsnapshotreader.h>
#include <vespa/config/print/fileconfigsnapshotwriter.h>
@@ -11,9 +12,7 @@
#include <vespa/config-summarymap.h>
#include <vespa/config-rank-profiles.h>
#include <vespa/searchsummary/config/config-juniperrc.h>
-#include <vespa/fastos/file.h>
#include <vespa/config/helper/configgetter.hpp>
-#include <fstream>
#include <sstream>
#include <fcntl.h>
@@ -360,8 +359,16 @@ FileConfigManager::loadConfig(const DocumentDBConfig &currentSnapshot,
* of default values here instead of the current values from the config
* server.
*/
+ const ProtonConfig &protonConfig = *_protonConfig;
+ const auto &hwDiskCfg = protonConfig.hwinfo.disk;
+ const auto &hwMemoryCfg = protonConfig.hwinfo.memory;
+ const auto &hwCpuCfg = protonConfig.hwinfo.cpu;
+ HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit,
+ hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores);
+ HwInfoSampler sampler(protonConfig.basedir, samplerCfg);
auto bootstrap = std::make_shared<BootstrapConfig>(1, docTypesCfg, repo, _protonConfig, filedistRpcConf,
- bucketspaces,currentSnapshot.getTuneFileDocumentDBSP());
+ bucketspaces,currentSnapshot.getTuneFileDocumentDBSP(),
+ sampler.hwInfo());
dbc.forwardConfig(bootstrap);
dbc.nextGeneration(0);
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 8ce7e0c79fe..2639958ed70 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -202,8 +202,6 @@ Proton::Proton(const config::ConfigUri & configUri,
_initStarted(false),
_initComplete(false),
_initDocumentDbsInSequence(false),
- _hwInfo(),
- _hwInfoSampler(),
_documentDBReferenceRegistry()
{
_documentDBReferenceRegistry = std::make_shared<DocumentDBReferenceRegistry>();
@@ -231,22 +229,12 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
{
assert( _initStarted && ! _initComplete );
const ProtonConfig &protonConfig = configSnapshot->getProtonConfig();
- const auto &hwDiskCfg = protonConfig.hwinfo.disk;
- const auto &hwMemoryCfg = protonConfig.hwinfo.memory;
- const auto &hwCpuCfg = protonConfig.hwinfo.cpu;
- HwInfoSampler::Config samplerCfg(hwDiskCfg.size,
- hwDiskCfg.writespeed,
- hwDiskCfg.slowwritespeedlimit,
- hwDiskCfg.samplewritesize,
- hwDiskCfg.shared,
- hwMemoryCfg.size,
- hwCpuCfg.cores);
- _hwInfoSampler = std::make_unique<HwInfoSampler>(protonConfig.basedir, samplerCfg);
- _hwInfo = _hwInfoSampler->hwInfo();
+ const HwInfo & hwInfo = configSnapshot->getHwInfo();
+
setFS4Compression(protonConfig);
_diskMemUsageSampler = std::make_unique<DiskMemUsageSampler>
(protonConfig.basedir,
- diskMemUsageSamplerConfig(protonConfig, _hwInfo));
+ diskMemUsageSamplerConfig(protonConfig, hwInfo));
_metricsEngine.reset(new MetricsEngine());
_metricsEngine->addMetricsHook(_metricsHook);
@@ -263,8 +251,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
switch (flush.strategy) {
case ProtonConfig::Flush::MEMORY: {
auto memoryFlush = std::make_shared<MemoryFlush>(
- MemoryFlushConfigUpdater::convertConfig(flush.memory, _hwInfo.memory()), fastos::ClockSystem::now());
- _memoryFlushConfigUpdater = std::make_unique<MemoryFlushConfigUpdater>(memoryFlush, flush.memory, _hwInfo.memory());
+ MemoryFlushConfigUpdater::convertConfig(flush.memory, hwInfo.memory()), fastos::ClockSystem::now());
+ _memoryFlushConfigUpdater = std::make_unique<MemoryFlushConfigUpdater>(memoryFlush, flush.memory, hwInfo.memory());
_diskMemUsageSampler->notifier().addDiskMemUsageListener(_memoryFlushConfigUpdater.get());
strategy = memoryFlush;
break;
@@ -296,7 +284,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
vespalib::string fileConfigId;
_warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024));
- const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, _hwInfo.cpu());
+ const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu());
_summaryExecutor.reset(new vespalib::BlockingThreadStackExecutor(summaryThreads, 128*1024, summaryThreads*16));
InitializeThreads initializeThreads;
if (protonConfig.initialize.threads > 0) {
@@ -351,7 +339,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot)
protonConfig.search.memory.limiter.minhits);
const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP();
- _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, _hwInfo));
+ _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, configSnapshot->getHwInfo()));
if (_memoryFlushConfigUpdater) {
_memoryFlushConfigUpdater->setConfig(protonConfig.flush.memory);
_flushEngine->kick();
@@ -545,7 +533,7 @@ Proton::addDocumentDB(const document::DocumentType &docType,
_fileHeaderContext,
std::move(config_store),
initializeThreads,
- _hwInfo));
+ bootstrapConfig->getHwInfo()));
try {
ret->start();
} catch (vespalib::Exception &e) {
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h
index 8d1026340ca..501b5ab8c62 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.h
@@ -12,7 +12,6 @@
#include "proton_configurer.h"
#include "rpc_hooks.h"
#include "bootstrapconfig.h"
-#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchcore/proton/flushengine/flushengine.h>
#include <vespa/searchcore/proton/matchengine/matchengine.h>
#include <vespa/searchcore/proton/matching/querylimiter.h>
@@ -38,7 +37,6 @@
namespace proton {
class DiskMemUsageSampler;
-class HwInfoSampler;
class IDocumentDBReferenceRegistry;
class Proton : public IProtonConfigurerOwner,
@@ -124,8 +122,6 @@ private:
bool _initStarted;
bool _initComplete;
bool _initDocumentDbsInSequence;
- HwInfo _hwInfo;
- std::unique_ptr<HwInfoSampler> _hwInfoSampler;
std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry;
IDocumentDBConfigOwner *
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp
index deeec695f26..f2deb456090 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp
@@ -65,8 +65,7 @@ ProtonConfigFetcher::pruneManagerMap(const BootstrapConfig::SP & config)
if (_dbManagerMap.find(docTypeName) != _dbManagerMap.end()) {
mgr = _dbManagerMap[docTypeName];
} else {
- mgr = DocumentDBConfigManager::SP(new DocumentDBConfigManager
- (ddb.configid, docTypeName.getName()));
+ mgr = std::make_shared<DocumentDBConfigManager>(ddb.configid, docTypeName.getName());
}
set.add(mgr->createConfigKeySet());
newMap[docTypeName] = mgr;
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h
index c8d1e55e4e4..8ba56deb7e8 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h
@@ -52,9 +52,9 @@ private:
using TimePoint = std::chrono::time_point<Clock>;
using OldDocumentTypeRepo = std::pair<TimePoint, std::shared_ptr<document::DocumentTypeRepo>>;
- BootstrapConfigManager _bootstrapConfigManager;
+ BootstrapConfigManager _bootstrapConfigManager;
config::ConfigRetriever _retriever;
- IProtonConfigurer & _owner;
+ IProtonConfigurer & _owner;
mutable std::mutex _mutex; // Protects maps
using lock_guard = std::lock_guard<std::mutex>;
@@ -71,6 +71,4 @@ private:
void rememberDocumentTypeRepo(std::shared_ptr<document::DocumentTypeRepo> repo);
};
-
} // namespace proton
-