summaryrefslogtreecommitdiffstats
path: root/searchcore
diff options
context:
space:
mode:
authorHarald Musum <musum@yahoo-inc.com>2018-01-31 18:10:17 +0100
committerGitHub <noreply@github.com>2018-01-31 18:10:17 +0100
commit3618b5fc5e3c843928aa1a79a2b3e077b8bf0881 (patch)
tree0173913d43d3d24f28aef7fd97db2ff006bd24d2 /searchcore
parent608db174b58e118adceeb38bda44d6b28f74b199 (diff)
Revert "Balder/compute summary cache automatically in backend"
Diffstat (limited to 'searchcore')
-rw-r--r--searchcore/src/apps/tests/persistenceconformance_test.cpp2
-rw-r--r--searchcore/src/tests/proton/docsummary/docsummary.cpp2
-rw-r--r--searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp2
-rw-r--r--searchcore/src/tests/proton/documentdb/documentdb_test.cpp2
-rw-r--r--searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp5
-rw-r--r--searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp57
-rw-r--r--searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/config/proton.def6
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/hw_info.h8
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp14
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h44
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp31
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h7
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp41
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp13
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/proton.cpp28
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/proton.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp3
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h6
20 files changed, 125 insertions, 156 deletions
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp
index 5b50ec49844..5c59b3376ba 100644
--- a/searchcore/src/apps/tests/persistenceconformance_test.cpp
+++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp
@@ -183,7 +183,7 @@ public:
std::make_shared<ProtonConfig>(),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(),
- tuneFileDocDB, HwInfo()));
+ tuneFileDocDB));
mgr.forwardConfig(b);
mgr.nextGeneration(0);
return DocumentDB::SP(
diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp
index ff393682f9b..dae50fcc860 100644
--- a/searchcore/src/tests/proton/docsummary/docsummary.cpp
+++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp
@@ -208,7 +208,7 @@ public:
std::make_shared<ProtonConfig>(),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(),
- _tuneFileDocumentDB, _hwInfo);
+ _tuneFileDocumentDB);
_configMgr.forwardConfig(b);
_configMgr.nextGeneration(0);
if (! FastOS_File::MakeDirectory((std::string("tmpdb/") + docTypeName).c_str())) { abort(); }
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index e39fd1fe28a..e655ae0d739 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -284,7 +284,7 @@ struct MyConfigSnapshot
std::make_shared<ProtonConfig>(),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(),
- tuneFileDocumentDB, HwInfo());
+ tuneFileDocumentDB);
config::DirSpec spec(cfgDir);
DocumentDBConfigHelper mgr(spec, "searchdocument");
mgr.forwardConfig(_bootstrap);
diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
index 9f6eb0ea4e1..f93930d189b 100644
--- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
@@ -99,7 +99,7 @@ Fixture::Fixture()
std::make_shared<ProtonConfig>(),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(),
- tuneFileDocumentDB, HwInfo()));
+ tuneFileDocumentDB));
mgr.forwardConfig(b);
mgr.nextGeneration(0);
_db.reset(new DocumentDB(".", mgr.getConfig(), "tcp/localhost:9014", _queryLimiter, _clock, DocTypeName("typea"),
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp
index dd35d823b99..6b1031fe558 100644
--- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp
@@ -40,12 +40,13 @@ makeBaseConfigSnapshot()
DBCM dbcm(spec, "test");
DocumenttypesConfigSP dtcfg(config::ConfigGetter<DocumenttypesConfig>::getConfig("", spec).release());
- BootstrapConfig::SP b(new BootstrapConfig(1, dtcfg,
+ BootstrapConfig::SP b(new BootstrapConfig(1,
+ dtcfg,
DocumentTypeRepo::SP(new DocumentTypeRepo(*dtcfg)),
std::make_shared<ProtonConfig>(),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(),
- std::make_shared<TuneFileDocumentDB>(), HwInfo()));
+ std::make_shared<TuneFileDocumentDB>()));
dbcm.forwardConfig(b);
dbcm.nextGeneration(0);
DocumentDBConfig::SP snap = dbcm.getConfig();
diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
index 96f742a172c..37e4dfc486d 100644
--- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
+++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp
@@ -14,12 +14,12 @@
#include <vespa/searchcore/proton/server/proton_config_fetcher.h>
#include <vespa/searchcore/proton/server/proton_config_snapshot.h>
#include <vespa/searchcore/proton/server/i_proton_configurer.h>
-#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchsummary/config/config-juniperrc.h>
#include <vespa/searchcore/config/config-ranking-constants.h>
#include <vespa/vespalib/testkit/testapp.h>
#include <vespa/vespalib/util/varholder.h>
#include <vespa/config-bucketspaces.h>
+#include <mutex>
using namespace config;
using namespace proton;
@@ -146,15 +146,14 @@ struct ConfigTestFixture {
documenttypesBuilder == bootstrapConfig->getDocumenttypesConfig());
}
- BootstrapConfig::SP getBootstrapConfig(int64_t generation, const HwInfo & hwInfo) const {
+ BootstrapConfig::SP getBootstrapConfig(int64_t generation) const {
return BootstrapConfig::SP(new BootstrapConfig(generation,
- std::make_shared<DocumenttypesConfig>(documenttypesBuilder),
- std::make_shared<DocumentTypeRepo>(documenttypesBuilder),
- std::make_shared<ProtonConfig>(protonBuilder),
+ BootstrapConfig::DocumenttypesConfigSP(new DocumenttypesConfig(documenttypesBuilder)),
+ DocumentTypeRepo::SP(new DocumentTypeRepo(documenttypesBuilder)),
+ BootstrapConfig::ProtonConfigSP(new ProtonConfig(protonBuilder)),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(bucketspacesBuilder),
- std::make_shared<TuneFileDocumentDB>(),
- hwInfo));
+ std::make_shared<TuneFileDocumentDB>()));
}
void reload() { context->reload(); }
@@ -233,20 +232,14 @@ TEST_FFF("require that bootstrap config manager updates config", ConfigTestFixtu
}
DocumentDBConfig::SP
-getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr, const HwInfo & hwInfo)
+getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr)
{
ConfigRetriever retriever(mgr.createConfigKeySet(), f.context);
- mgr.forwardConfig(f.getBootstrapConfig(1, hwInfo));
+ mgr.forwardConfig(f.getBootstrapConfig(1));
mgr.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs
return mgr.getConfig();
}
-DocumentDBConfig::SP
-getDocumentDBConfig(ConfigTestFixture &f, DocumentDBConfigManager &mgr)
-{
- return getDocumentDBConfig(f, mgr, HwInfo());
-}
-
TEST_FF("require that documentdb config manager subscribes for config",
ConfigTestFixture("search"),
DocumentDBConfigManager(f1.configId + "/typea", "typea")) {
@@ -358,38 +351,4 @@ TEST_FF("require that prune removed documents interval can be set based on age",
EXPECT_EQUAL(20, config->getMaintenanceConfigSP()->getPruneRemovedDocumentsConfig().getInterval());
}
-TEST_FF("require that docstore config computes cachesize automatically if unset",
- ConfigTestFixture("test"),
- DocumentDBConfigManager(f1.configId + "/test", "test"))
-{
- HwInfo hwInfo(HwInfo::Disk(1, false, false), HwInfo::Memory(1000000), HwInfo::Cpu(1));
- f1.addDocType("test");
- f1.protonBuilder.summary.cache.maxbytes = 2000;
- auto config = getDocumentDBConfig(f1, f2, hwInfo);
- EXPECT_EQUAL(2000ul, config->getStoreConfig().getMaxCacheBytes());
-
- f1.protonBuilder.summary.cache.maxbytes = -7;
- config = getDocumentDBConfig(f1, f2, hwInfo);
- EXPECT_EQUAL(70000ul, config->getStoreConfig().getMaxCacheBytes());
-
- f1.protonBuilder.summary.cache.maxbytes = -700;
- config = getDocumentDBConfig(f1, f2, hwInfo);
- EXPECT_EQUAL(500000ul, config->getStoreConfig().getMaxCacheBytes());
-}
-
-TEST("test HwInfo equality") {
- EXPECT_TRUE(HwInfo::Cpu(1) == HwInfo::Cpu(1));
- EXPECT_FALSE(HwInfo::Cpu(1) == HwInfo::Cpu(2));
- EXPECT_TRUE(HwInfo::Memory(1) == HwInfo::Memory(1));
- EXPECT_FALSE(HwInfo::Memory(1) == HwInfo::Memory(2));
- EXPECT_TRUE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, false,false));
- EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, false,true));
- EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(1, true,false));
- EXPECT_FALSE(HwInfo::Disk(1, false, false) == HwInfo::Disk(2, false,false));
- EXPECT_TRUE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 1ul, 1ul));
- EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 1ul, 2ul));
- EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(1, false,false), 2ul, 1ul));
- EXPECT_FALSE(HwInfo(HwInfo::Disk(1, false, false), 1ul, 1ul) == HwInfo(HwInfo::Disk(2, false,false), 1ul, 1ul));
-}
-
TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
index d59a67d73bb..a8336e29f9f 100644
--- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
+++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
@@ -171,7 +171,7 @@ struct ConfigFixture {
BootstrapConfig::ProtonConfigSP(new ProtonConfig(_protonBuilder)),
std::make_shared<FiledistributorrpcConfig>(),
std::make_shared<BucketspacesConfig>(_bucketspacesBuilder),
- std::make_shared<TuneFileDocumentDB>(), HwInfo()));
+ std::make_shared<TuneFileDocumentDB>()));
}
std::shared_ptr<ProtonConfigSnapshot> getConfigSnapshot()
diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def
index c7a3fc7cb62..d7b1cbfa45f 100644
--- a/searchcore/src/vespa/searchcore/config/proton.def
+++ b/searchcore/src/vespa/searchcore/config/proton.def
@@ -2,7 +2,7 @@
namespace=vespa.config.search.core
## Base directory. The default is ignored as it is assigned by the model
-basedir string default="." restart
+basedir string default="tmp" restart
## specifies the port number for the persistent internal transport
## protocol provided for a multi-level dispatch system.
@@ -210,9 +210,7 @@ grow.numdocs int default=10000 restart
grow.multivalueallocfactor double default=0.2 restart
## Control cache size in bytes.
-## Postive numbers are absolute in bytes.
-## Negative numbers are a percentage of memory.
-summary.cache.maxbytes long default=-5
+summary.cache.maxbytes long default=0
## Include visits in the cache, if the visitoperation allows it.
## This will enable another separate cache of summary.cache.maxbytes size.
diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info.h b/searchcore/src/vespa/searchcore/proton/common/hw_info.h
index 9129aa43483..cd0ef2817d7 100644
--- a/searchcore/src/vespa/searchcore/proton/common/hw_info.h
+++ b/searchcore/src/vespa/searchcore/proton/common/hw_info.h
@@ -23,9 +23,6 @@ public:
uint64_t sizeBytes() const { return _sizeBytes; }
bool slow() const { return _slow; }
bool shared() const { return _shared; }
- bool operator == (const Disk & rhs) const {
- return (_sizeBytes == rhs._sizeBytes) && (_slow == rhs._slow) && (_shared == rhs._shared);
- }
};
class Memory {
@@ -34,7 +31,6 @@ public:
public:
Memory(uint64_t sizeBytes_) : _sizeBytes(sizeBytes_) {}
uint64_t sizeBytes() const { return _sizeBytes; }
- bool operator == (const Memory & rhs) const { return _sizeBytes == rhs._sizeBytes; }
};
class Cpu {
@@ -43,7 +39,6 @@ public:
public:
Cpu(uint32_t cores_) : _cores(cores_) {}
uint32_t cores() const { return _cores; }
- bool operator == (const Cpu & rhs) const { return _cores == rhs._cores; }
};
private:
@@ -71,9 +66,6 @@ public:
const Disk &disk() const { return _disk; }
const Memory &memory() const { return _memory; }
const Cpu &cpu() const { return _cpu; }
- bool operator == (const HwInfo & rhs) const {
- return (_cpu == rhs._cpu) && (_disk == rhs._disk) && (_memory == rhs._memory);
- }
};
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp
index 1bc5cf00ad0..27d430b2893 100644
--- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp
@@ -29,30 +29,29 @@ BootstrapConfig::BootstrapConfig(
const ProtonConfigSP &protonConfig,
const FiledistributorrpcConfigSP &filedistRpcConfSP,
const BucketspacesConfigSP &bucketspaces,
- const search::TuneFileDocumentDB::SP &tuneFileDocumentDB,
- const HwInfo & hwInfo)
+ const search::TuneFileDocumentDB::SP &tuneFileDocumentDB)
: _documenttypes(documenttypes),
_repo(repo),
_proton(protonConfig),
_fileDistributorRpc(filedistRpcConfSP),
_bucketspaces(bucketspaces),
_tuneFileDocumentDB(tuneFileDocumentDB),
- _hwInfo(hwInfo),
_generation(generation)
{ }
-BootstrapConfig::~BootstrapConfig() = default;
+BootstrapConfig::~BootstrapConfig() { }
bool
BootstrapConfig::operator==(const BootstrapConfig &rhs) const
{
- return equals<DocumenttypesConfig>(_documenttypes.get(), rhs._documenttypes.get()) &&
+ return equals<DocumenttypesConfig>(_documenttypes.get(),
+ rhs._documenttypes.get()) &&
_repo.get() == rhs._repo.get() &&
equals<ProtonConfig>(_proton.get(), rhs._proton.get()) &&
equals<FiledistributorrpcConfig>(_fileDistributorRpc.get(), rhs._fileDistributorRpc.get()) &&
equals<BucketspacesConfig>(_bucketspaces.get(), rhs._bucketspaces.get()) &&
- equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(), rhs._tuneFileDocumentDB.get()) &&
- (_hwInfo == rhs._hwInfo);
+ equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(),
+ rhs._tuneFileDocumentDB.get());
}
@@ -62,4 +61,5 @@ BootstrapConfig::valid() const
return _documenttypes && _repo && _proton && _fileDistributorRpc && _bucketspaces && _tuneFileDocumentDB;
}
+
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h
index 368cc30d8a4..1f888ac1739 100644
--- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h
@@ -3,11 +3,11 @@
#pragma once
#include "documentdbconfig.h"
-#include <vespa/searchcore/proton/common/hw_info.h>
-#include <vespa/searchcore/config/config-proton.h>
#include <vespa/document/config/config-documenttypes.h>
#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcore/config/config-proton.h>
#include <vespa/searchlib/common/tunefileinfo.h>
+#include <vespa/config/retriever/configkeyset.h>
#include <vespa/config/retriever/configsnapshot.h>
#include <vespa/fileacquirer/config-filedistributorrpc.h>
@@ -37,7 +37,6 @@ private:
FiledistributorrpcConfigSP _fileDistributorRpc;
BucketspacesConfigSP _bucketspaces;
search::TuneFileDocumentDB::SP _tuneFileDocumentDB;
- HwInfo _hwInfo;
int64_t _generation;
public:
@@ -47,21 +46,37 @@ public:
const ProtonConfigSP &protonConfig,
const FiledistributorrpcConfigSP &filedistRpcConfSP,
const BucketspacesConfigSP &bucketspaces,
- const search::TuneFileDocumentDB::SP &_tuneFileDocumentDB,
- const HwInfo & hwInfo);
+ const search::TuneFileDocumentDB::SP &
+ _tuneFileDocumentDB);
~BootstrapConfig();
- const document::DocumenttypesConfig &getDocumenttypesConfig() const { return *_documenttypes; }
- const FiledistributorrpcConfig &getFiledistributorrpcConfig() const { return *_fileDistributorRpc; }
- const FiledistributorrpcConfigSP &getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; }
- const DocumenttypesConfigSP &getDocumenttypesConfigSP() const { return _documenttypes; }
- const document::DocumentTypeRepo::SP &getDocumentTypeRepoSP() const { return _repo; }
- const vespa::config::search::core::ProtonConfig &getProtonConfig() const { return *_proton; }
- const ProtonConfigSP &getProtonConfigSP() const { return _proton; }
+ const document::DocumenttypesConfig &
+ getDocumenttypesConfig() const { return *_documenttypes; }
+
+ const cloud::config::filedistribution::FiledistributorrpcConfig &
+ getFiledistributorrpcConfig() const { return *_fileDistributorRpc; }
+
+ const FiledistributorrpcConfigSP &
+ getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; }
+
+ const DocumenttypesConfigSP &
+ getDocumenttypesConfigSP() const { return _documenttypes; }
+
+ const document::DocumentTypeRepo::SP &
+ getDocumentTypeRepoSP() const { return _repo; }
+
+ const vespa::config::search::core::ProtonConfig &
+ getProtonConfig() const { return *_proton; }
+
+ const ProtonConfigSP &
+ getProtonConfigSP() const { return _proton; }
+
const BucketspacesConfigSP &getBucketspacesConfigSP() const { return _bucketspaces; }
- const search::TuneFileDocumentDB::SP &getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; }
+
+ const search::TuneFileDocumentDB::SP &
+ getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; }
+
int64_t getGeneration() const { return _generation; }
- const HwInfo & getHwInfo() const { return _hwInfo; }
/**
* Shared pointers are checked for identity, not equality.
@@ -71,3 +86,4 @@ public:
};
} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp
index 5459d60e769..9088a0b3b88 100644
--- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp
@@ -2,7 +2,6 @@
#include "bootstrapconfigmanager.h"
#include "bootstrapconfig.h"
-#include <vespa/searchcore/proton/common/hw_info_sampler.h>
#include <vespa/config-bucketspaces.h>
#include <vespa/searchlib/common/tunefileinfo.hpp>
@@ -39,13 +38,6 @@ BootstrapConfigManager::createConfigKeySet() const
.add<BucketspacesConfig>(_configId);
}
-std::shared_ptr<BootstrapConfig>
-BootstrapConfigManager::getConfig() const
-{
- std::lock_guard<std::mutex> lock(_pendingConfigMutex);
- return _pendingConfigSnapshot;
-}
-
void
BootstrapConfigManager::update(const ConfigSnapshot & snapshot)
{
@@ -91,17 +83,21 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot)
if (snapshot.isChanged<FiledistributorrpcConfig>(_configId, currentGen)) {
LOG(info, "Filedistributorrpc config is changed");
- newFiledistRpcConfSP = snapshot.getConfig<FiledistributorrpcConfig>(_configId);
+ auto p = snapshot.getConfig<FiledistributorrpcConfig>(_configId);
+ newFiledistRpcConfSP = BootstrapConfig::FiledistributorrpcConfigSP(std::move(p));
}
if (snapshot.isChanged<DocumenttypesConfig>(_configId, currentGen)) {
LOG(spam, "Documenttypes config is changed");
- newDocumenttypesConfig = snapshot.getConfig<DocumenttypesConfig>(_configId);
- newRepo = std::make_shared<DocumentTypeRepo>(*newDocumenttypesConfig);
+ std::unique_ptr<DocumenttypesConfig> documenttypesConfig = snapshot.getConfig<DocumenttypesConfig>(_configId);
+ DocumentTypeRepo::SP repo(new DocumentTypeRepo(*documenttypesConfig));
+ newDocumenttypesConfig = DocumenttypesConfigSP(documenttypesConfig.release());
+ newRepo = repo;
}
if (snapshot.isChanged<BucketspacesConfig>(_configId, currentGen)) {
LOG(spam, "Bucketspaces config is changed");
- newBucketspacesConfig = snapshot.getConfig<BucketspacesConfig>(_configId);
+ std::unique_ptr<BucketspacesConfig> bucketspacesConfig = snapshot.getConfig<BucketspacesConfig>(_configId);
+ newBucketspacesConfig = std::move(bucketspacesConfig);
}
assert(newProtonConfig);
assert(newFiledistRpcConfSP);
@@ -110,17 +106,8 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot)
assert(newDocumenttypesConfig);
assert(newRepo);
- const ProtonConfig &protonConfig = *newProtonConfig;
- const auto &hwDiskCfg = protonConfig.hwinfo.disk;
- const auto &hwMemoryCfg = protonConfig.hwinfo.memory;
- const auto &hwCpuCfg = protonConfig.hwinfo.cpu;
- HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit,
- hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores);
- HwInfoSampler sampler(protonConfig.basedir, samplerCfg);
-
auto newSnapshot(std::make_shared<BootstrapConfig>(snapshot.getGeneration(), newDocumenttypesConfig, newRepo,
- newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig,
- newTuneFileDocumentDB, sampler.hwInfo()));
+ newProtonConfig, newFiledistRpcConfSP, newBucketspacesConfig, newTuneFileDocumentDB));
assert(newSnapshot->valid());
{
diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h
index 0b0dd341975..3862f11cdd9 100644
--- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h
+++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.h
@@ -20,7 +20,12 @@ public:
~BootstrapConfigManager();
const config::ConfigKeySet createConfigKeySet() const;
- std::shared_ptr<BootstrapConfig> getConfig() const;
+ std::shared_ptr<BootstrapConfig>
+ getConfig() const
+ {
+ std::lock_guard<std::mutex> lock(_pendingConfigMutex);
+ return _pendingConfigSnapshot;
+ }
void update(const config::ConfigSnapshot & snapshot);
private:
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
index ec22d3293c4..def18b76360 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
@@ -2,8 +2,6 @@
#include "documentdbconfigmanager.h"
#include "bootstrapconfig.h"
-#include <vespa/searchcore/proton/common/hw_info.h>
-#include <vespa/searchcore/config/config-ranking-constants.h>
#include <vespa/config-imported-fields.h>
#include <vespa/config-rank-profiles.h>
#include <vespa/config-summarymap.h>
@@ -12,6 +10,7 @@
#include <vespa/searchcommon/common/schemaconfigurer.h>
#include <vespa/searchlib/index/schemautil.h>
#include <vespa/searchsummary/config/config-juniperrc.h>
+#include <vespa/searchcore/config/config-ranking-constants.h>
#include <vespa/vespalib/time/time_box.h>
#include <vespa/log/log.h>
@@ -158,17 +157,14 @@ deriveCompression(const T & config) {
}
DocumentStore::Config
-getStoreConfig(const ProtonConfig::Summary::Cache & cache, const HwInfo & hwInfo)
+getStoreConfig(const ProtonConfig::Summary::Cache & cache)
{
- size_t maxBytes = (cache.maxbytes < 0)
- ? (hwInfo.memory().sizeBytes()*std::min(50l, -cache.maxbytes))/100l
- : cache.maxbytes;
- return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching);
+ return DocumentStore::Config(deriveCompression(cache.compression), cache.maxbytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching);
}
LogDocumentStore::Config
-deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush, const HwInfo & hwInfo) {
- DocumentStore::Config config(getStoreConfig(summary.cache, hwInfo));
+deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::Memory & flush) {
+ DocumentStore::Config config(getStoreConfig(summary.cache));
const ProtonConfig::Summary::Log & log(summary.log);
const ProtonConfig::Summary::Log::Chunk & chunk(log.chunk);
WriteableFileChunk::Config fileConfig(deriveCompression(chunk.compression), chunk.maxbytes);
@@ -181,8 +177,8 @@ deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::M
return LogDocumentStore::Config(config, logConfig);
}
-search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton, const HwInfo & hwInfo) {
- return deriveConfig(proton.summary, proton.flush.memory, hwInfo);
+search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton) {
+ return deriveConfig(proton.summary, proton.flush.memory);
}
using AttributesConfigSP = DocumentDBConfig::AttributesConfigSP;
@@ -259,7 +255,10 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot)
}
if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration)) {
- newRankProfilesConfig = snapshot.getConfig<RankProfilesConfig>(_configId);
+ newRankProfilesConfig =
+ RankProfilesConfigSP(
+ snapshot.getConfig<RankProfilesConfig>(_configId).
+ release());
}
if (snapshot.isChanged<RankingConstantsConfig>(_configId, currentGeneration)) {
RankingConstantsConfigSP newRankingConstantsConfig = RankingConstantsConfigSP(
@@ -287,34 +286,34 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot)
newRankingConstants = std::make_shared<RankingConstants>(constants);
}
if (snapshot.isChanged<IndexschemaConfig>(_configId, currentGeneration)) {
- newIndexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId);
+ std::unique_ptr<IndexschemaConfig> indexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId);
search::index::Schema schema;
- search::index::SchemaBuilder::build(*newIndexschemaConfig, schema);
+ search::index::SchemaBuilder::build(*indexschemaConfig, schema);
if (!search::index::SchemaUtil::validateSchema(schema)) {
LOG(error, "Cannot use bad index schema, validation failed");
abort();
}
+ newIndexschemaConfig = IndexschemaConfigSP(indexschemaConfig.release());
}
if (snapshot.isChanged<AttributesConfig>(_configId, currentGeneration)) {
- newAttributesConfig = snapshot.getConfig<AttributesConfig>(_configId);
+ newAttributesConfig = AttributesConfigSP(snapshot.getConfig<AttributesConfig>(_configId).release());
}
if (snapshot.isChanged<SummaryConfig>(_configId, currentGeneration)) {
- newSummaryConfig = snapshot.getConfig<SummaryConfig>(_configId);
+ newSummaryConfig = SummaryConfigSP(snapshot.getConfig<SummaryConfig>(_configId).release());
}
if (snapshot.isChanged<SummarymapConfig>(_configId, currentGeneration)) {
- newSummarymapConfig = snapshot.getConfig<SummarymapConfig>(_configId);
+ newSummarymapConfig = SummarymapConfigSP(snapshot.getConfig<SummarymapConfig>(_configId).release());
}
if (snapshot.isChanged<JuniperrcConfig>(_configId, currentGeneration)) {
- newJuniperrcConfig = snapshot.getConfig<JuniperrcConfig>(_configId);
+ newJuniperrcConfig = JuniperrcConfigSP(snapshot.getConfig<JuniperrcConfig>(_configId).release());
}
if (snapshot.isChanged<ImportedFieldsConfig>(_configId, currentGeneration)) {
- newImportedFieldsConfig = snapshot.getConfig<ImportedFieldsConfig>(_configId);
+ newImportedFieldsConfig = ImportedFieldsConfigSP(snapshot.getConfig<ImportedFieldsConfig>(_configId).release());
}
Schema::SP schema(buildSchema(*newAttributesConfig, *newSummaryConfig, *newIndexschemaConfig));
newMaintenanceConfig = buildMaintenanceConfig(_bootstrapConfig, _docTypeName);
- search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig(),
- _bootstrapConfig->getHwInfo());
+ search::LogDocumentStore::Config storeConfig = buildStoreConfig(_bootstrapConfig->getProtonConfig());
if (newMaintenanceConfig && oldMaintenanceConfig && *newMaintenanceConfig == *oldMaintenanceConfig) {
newMaintenanceConfig = oldMaintenanceConfig;
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h
index bd1cb66de43..80654c7588b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h
@@ -2,14 +2,14 @@
#pragma once
-#include "documentdbconfig.h"
+#include <vespa/vespalib/stllike/string.h>
#include <vespa/config/config.h>
+#include "documentdbconfig.h"
#include <mutex>
namespace proton {
class BootstrapConfig;
-
/**
* This class manages the subscription for documentdb configs.
*/
diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
index 37a8ee2aadf..cfb96172800 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
@@ -2,7 +2,6 @@
#include "fileconfigmanager.h"
#include "bootstrapconfig.h"
-#include <vespa/searchcore/proton/common/hw_info_sampler.h>
#include <vespa/config/print/fileconfigwriter.h>
#include <vespa/config/print/fileconfigsnapshotreader.h>
#include <vespa/config/print/fileconfigsnapshotwriter.h>
@@ -12,7 +11,9 @@
#include <vespa/config-summarymap.h>
#include <vespa/config-rank-profiles.h>
#include <vespa/searchsummary/config/config-juniperrc.h>
+#include <vespa/fastos/file.h>
#include <vespa/config/helper/configgetter.hpp>
+#include <fstream>
#include <sstream>
#include <fcntl.h>
@@ -359,16 +360,8 @@ FileConfigManager::loadConfig(const DocumentDBConfig &currentSnapshot,
* of default values here instead of the current values from the config
* server.
*/
- const ProtonConfig &protonConfig = *_protonConfig;
- const auto &hwDiskCfg = protonConfig.hwinfo.disk;
- const auto &hwMemoryCfg = protonConfig.hwinfo.memory;
- const auto &hwCpuCfg = protonConfig.hwinfo.cpu;
- HwInfoSampler::Config samplerCfg(hwDiskCfg.size, hwDiskCfg.writespeed, hwDiskCfg.slowwritespeedlimit,
- hwDiskCfg.samplewritesize, hwDiskCfg.shared, hwMemoryCfg.size, hwCpuCfg.cores);
- HwInfoSampler sampler(protonConfig.basedir, samplerCfg);
auto bootstrap = std::make_shared<BootstrapConfig>(1, docTypesCfg, repo, _protonConfig, filedistRpcConf,
- bucketspaces,currentSnapshot.getTuneFileDocumentDBSP(),
- sampler.hwInfo());
+ bucketspaces,currentSnapshot.getTuneFileDocumentDBSP());
dbc.forwardConfig(bootstrap);
dbc.nextGeneration(0);
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 2639958ed70..8ce7e0c79fe 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -202,6 +202,8 @@ Proton::Proton(const config::ConfigUri & configUri,
_initStarted(false),
_initComplete(false),
_initDocumentDbsInSequence(false),
+ _hwInfo(),
+ _hwInfoSampler(),
_documentDBReferenceRegistry()
{
_documentDBReferenceRegistry = std::make_shared<DocumentDBReferenceRegistry>();
@@ -229,12 +231,22 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
{
assert( _initStarted && ! _initComplete );
const ProtonConfig &protonConfig = configSnapshot->getProtonConfig();
- const HwInfo & hwInfo = configSnapshot->getHwInfo();
-
+ const auto &hwDiskCfg = protonConfig.hwinfo.disk;
+ const auto &hwMemoryCfg = protonConfig.hwinfo.memory;
+ const auto &hwCpuCfg = protonConfig.hwinfo.cpu;
+ HwInfoSampler::Config samplerCfg(hwDiskCfg.size,
+ hwDiskCfg.writespeed,
+ hwDiskCfg.slowwritespeedlimit,
+ hwDiskCfg.samplewritesize,
+ hwDiskCfg.shared,
+ hwMemoryCfg.size,
+ hwCpuCfg.cores);
+ _hwInfoSampler = std::make_unique<HwInfoSampler>(protonConfig.basedir, samplerCfg);
+ _hwInfo = _hwInfoSampler->hwInfo();
setFS4Compression(protonConfig);
_diskMemUsageSampler = std::make_unique<DiskMemUsageSampler>
(protonConfig.basedir,
- diskMemUsageSamplerConfig(protonConfig, hwInfo));
+ diskMemUsageSamplerConfig(protonConfig, _hwInfo));
_metricsEngine.reset(new MetricsEngine());
_metricsEngine->addMetricsHook(_metricsHook);
@@ -251,8 +263,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
switch (flush.strategy) {
case ProtonConfig::Flush::MEMORY: {
auto memoryFlush = std::make_shared<MemoryFlush>(
- MemoryFlushConfigUpdater::convertConfig(flush.memory, hwInfo.memory()), fastos::ClockSystem::now());
- _memoryFlushConfigUpdater = std::make_unique<MemoryFlushConfigUpdater>(memoryFlush, flush.memory, hwInfo.memory());
+ MemoryFlushConfigUpdater::convertConfig(flush.memory, _hwInfo.memory()), fastos::ClockSystem::now());
+ _memoryFlushConfigUpdater = std::make_unique<MemoryFlushConfigUpdater>(memoryFlush, flush.memory, _hwInfo.memory());
_diskMemUsageSampler->notifier().addDiskMemUsageListener(_memoryFlushConfigUpdater.get());
strategy = memoryFlush;
break;
@@ -284,7 +296,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
vespalib::string fileConfigId;
_warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024));
- const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu());
+ const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, _hwInfo.cpu());
_summaryExecutor.reset(new vespalib::BlockingThreadStackExecutor(summaryThreads, 128*1024, summaryThreads*16));
InitializeThreads initializeThreads;
if (protonConfig.initialize.threads > 0) {
@@ -339,7 +351,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot)
protonConfig.search.memory.limiter.minhits);
const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP();
- _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, configSnapshot->getHwInfo()));
+ _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, _hwInfo));
if (_memoryFlushConfigUpdater) {
_memoryFlushConfigUpdater->setConfig(protonConfig.flush.memory);
_flushEngine->kick();
@@ -533,7 +545,7 @@ Proton::addDocumentDB(const document::DocumentType &docType,
_fileHeaderContext,
std::move(config_store),
initializeThreads,
- bootstrapConfig->getHwInfo()));
+ _hwInfo));
try {
ret->start();
} catch (vespalib::Exception &e) {
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h
index 501b5ab8c62..8d1026340ca 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.h
@@ -12,6 +12,7 @@
#include "proton_configurer.h"
#include "rpc_hooks.h"
#include "bootstrapconfig.h"
+#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchcore/proton/flushengine/flushengine.h>
#include <vespa/searchcore/proton/matchengine/matchengine.h>
#include <vespa/searchcore/proton/matching/querylimiter.h>
@@ -37,6 +38,7 @@
namespace proton {
class DiskMemUsageSampler;
+class HwInfoSampler;
class IDocumentDBReferenceRegistry;
class Proton : public IProtonConfigurerOwner,
@@ -122,6 +124,8 @@ private:
bool _initStarted;
bool _initComplete;
bool _initDocumentDbsInSequence;
+ HwInfo _hwInfo;
+ std::unique_ptr<HwInfoSampler> _hwInfoSampler;
std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry;
IDocumentDBConfigOwner *
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp
index f2deb456090..deeec695f26 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp
@@ -65,7 +65,8 @@ ProtonConfigFetcher::pruneManagerMap(const BootstrapConfig::SP & config)
if (_dbManagerMap.find(docTypeName) != _dbManagerMap.end()) {
mgr = _dbManagerMap[docTypeName];
} else {
- mgr = std::make_shared<DocumentDBConfigManager>(ddb.configid, docTypeName.getName());
+ mgr = DocumentDBConfigManager::SP(new DocumentDBConfigManager
+ (ddb.configid, docTypeName.getName()));
}
set.add(mgr->createConfigKeySet());
newMap[docTypeName] = mgr;
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h
index 8ba56deb7e8..c8d1e55e4e4 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h
@@ -52,9 +52,9 @@ private:
using TimePoint = std::chrono::time_point<Clock>;
using OldDocumentTypeRepo = std::pair<TimePoint, std::shared_ptr<document::DocumentTypeRepo>>;
- BootstrapConfigManager _bootstrapConfigManager;
+ BootstrapConfigManager _bootstrapConfigManager;
config::ConfigRetriever _retriever;
- IProtonConfigurer & _owner;
+ IProtonConfigurer & _owner;
mutable std::mutex _mutex; // Protects maps
using lock_guard = std::lock_guard<std::mutex>;
@@ -71,4 +71,6 @@ private:
void rememberDocumentTypeRepo(std::shared_ptr<document::DocumentTypeRepo> repo);
};
+
} // namespace proton
+