diff options
author | Tor Brede Vekterli <vekterli@vespa.ai> | 2024-05-14 12:31:29 +0000 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@vespa.ai> | 2024-05-15 15:04:35 +0000 |
commit | 1e234886e11e1d4b78507d240e3bd7234f203466 (patch) | |
tree | a229b74828312ee8ee84d9408e603d3df684ed70 | |
parent | fc104fab59e2c4b74e07e1c5f2c253eea7efacfc (diff) |
Remove usages of deprecated DirConfig in storage unit tests
Introduce a distinct `StorageConfigSet` which wraps the actual
underlying config objects and exposes them through a unified
`ConfigUri`.
38 files changed, 551 insertions, 282 deletions
diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index b5501d38241..437e8a14e3b 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -3,6 +3,7 @@ #include <tests/common/dummystoragelink.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> +#include <tests/common/storage_config_set.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/datatype/documenttype.h> @@ -23,7 +24,6 @@ #include <vespa/vdslib/state/random.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/config-stor-filestor.h> #include <future> #include <vespa/log/log.h> @@ -59,6 +59,7 @@ struct TestParams; struct BucketManagerTest : public Test { public: + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestServiceLayerApp> _node; std::unique_ptr<DummyStorageLink> _top; BucketManager *_manager; @@ -124,26 +125,18 @@ BucketManagerTest::~BucketManagerTest() = default; FAIL() << ost.str(); \ } -std::string getMkDirDisk(const std::string & rootFolder, int disk) { - std::ostringstream os; - os << "mkdir -p " << rootFolder << "/disks/d" << disk; - return os.str(); -} - void BucketManagerTest::setupTestEnvironment() { - vdstestlib::DirConfig config(getStandardConfig(true, "bucketmanagertest")); - + _config = StorageConfigSet::make_storage_node_config(); auto repo = std::make_shared<const DocumentTypeRepo>( *ConfigGetter<DocumenttypesConfig>::getConfig("config-doctypes", FileSpec("../config-doctypes.cfg"))); _top = std::make_unique<DummyStorageLink>(); - _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), config.getConfigId()); + _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), _config->config_uri()); _node->setTypeRepo(repo); _node->setupDummyPersistence(); // Set up the 3 links - auto config_uri = config::ConfigUri(config.getConfigId()); using vespa::config::content::core::StorServerConfig; - auto manager = std::make_unique<BucketManager>(*config_from<StorServerConfig>(config_uri), _node->getComponentRegister()); + auto manager = std::make_unique<BucketManager>(*config_from<StorServerConfig>(_config->config_uri()), _node->getComponentRegister()); _manager = manager.get(); _top->push_back(std::move(manager)); auto bottom = std::make_unique<DummyStorageLink>(); diff --git a/storage/src/tests/common/CMakeLists.txt b/storage/src/tests/common/CMakeLists.txt index 36487660cce..14cb510499f 100644 --- a/storage/src/tests/common/CMakeLists.txt +++ b/storage/src/tests/common/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(storage_testcommon TEST SOURCES dummystoragelink.cpp message_sender_stub.cpp + storage_config_set.cpp testhelper.cpp testnodestateupdater.cpp teststorageapp.cpp diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 7216bef03db..6ca84f0304a 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -1,5 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> #include <vespa/storage/bucketdb/bucketmanager.h> @@ -7,9 +11,6 @@ #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/persistence/filestorage/filestormetrics.h> #include <vespa/storage/visiting/visitormetrics.h> -#include <tests/common/teststorageapp.h> -#include <tests/common/testhelper.h> -#include <tests/common/dummystoragelink.h> #include <vespa/metrics/metricmanager.h> #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/gtest/gtest.h> @@ -26,10 +27,10 @@ namespace storage { struct MetricsTest : public Test { framework::defaultimplementation::FakeClock* _clock; + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestServiceLayerApp> _node; std::unique_ptr<DummyStorageLink> _top; std::unique_ptr<StatusMetricConsumer> _metricsConsumer; - std::unique_ptr<vdstestlib::DirConfig> _config; std::unique_ptr<metrics::MetricSet> _topSet; std::unique_ptr<metrics::MetricManager> _metricManager; std::shared_ptr<FileStorMetrics> _filestorMetrics; @@ -66,8 +67,8 @@ MetricsTest::MetricsTest() MetricsTest::~MetricsTest() = default; void MetricsTest::SetUp() { - _config = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true, "metricstest")); - _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), _config->getConfigId()); + _config = StorageConfigSet::make_storage_node_config(); + _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), _config->config_uri()); _node->setupDummyPersistence(); _clock = &_node->getClock(); _clock->setAbsoluteTimeInSeconds(1000000); @@ -92,7 +93,7 @@ void MetricsTest::SetUp() { _visitorMetrics = std::make_shared<VisitorMetrics>(); _visitorMetrics->initThreads(4); _topSet->registerMetric(*_visitorMetrics); - _metricManager->init(config::ConfigUri(_config->getConfigId())); + _metricManager->init(_config->config_uri()); } void MetricsTest::TearDown() { diff --git a/storage/src/tests/common/storage_config_set.cpp b/storage/src/tests/common/storage_config_set.cpp new file mode 100644 index 00000000000..ed10e113867 --- /dev/null +++ b/storage/src/tests/common/storage_config_set.cpp @@ -0,0 +1,153 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "storage_config_set.h" +#include <vespa/config-bucketspaces.h> +#include <vespa/config-persistence.h> +#include <vespa/config-slobroks.h> +#include <vespa/config-stor-distribution.h> +#include <vespa/config-stor-filestor.h> +#include <vespa/config-upgrading.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/document_type_repo_factory.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/messagebus/config-messagebus.h> +#include <vespa/messagebus/testlib/slobrok.h> +#include <vespa/metrics/config-metricsmanager.h> +#include <vespa/storage/config/config-stor-bouncer.h> +#include <vespa/storage/config/config-stor-communicationmanager.h> +#include <vespa/storage/config/config-stor-distributormanager.h> +#include <vespa/storage/config/config-stor-prioritymapping.h> +#include <vespa/storage/config/config-stor-server.h> +#include <vespa/storage/config/config-stor-status.h> +#include <vespa/storage/config/config-stor-visitordispatcher.h> +#include <vespa/storage/visiting/config-stor-visitor.h> +#include <vespa/document/base/testdocrepo.h> +#include <vespa/vespalib/util/stringfmt.h> + +namespace storage { + +StorageConfigSet::StorageConfigSet(vespalib::string config_id_str, bool is_storage_node) + : _document_type_config(std::make_unique<DocumenttypesConfigBuilder>()), + _slobroks_config(std::make_unique<SlobroksConfigBuilder>()), + _messagebus_config(std::make_unique<MessagebusConfigBuilder>()), + _metrics_config(std::make_unique<MetricsmanagerConfigBuilder>()), + _persistence_config(std::make_unique<PersistenceConfigBuilder>()), + _distribution_config(std::make_unique<StorDistributionConfigBuilder>()), + _filestor_config(std::make_unique<StorFilestorConfigBuilder>()), + _upgrading_config(std::make_unique<UpgradingConfigBuilder>()), + _bucket_spaces_config(std::make_unique<BucketspacesConfigBuilder>()), + _bouncer_config(std::make_unique<StorBouncerConfigBuilder>()), + _communication_manager_config(std::make_unique<StorCommunicationmanagerConfigBuilder>()), + _distributor_manager_config(std::make_unique<StorDistributormanagerConfigBuilder>()), + _priority_mapping_config(std::make_unique<StorPrioritymappingConfigBuilder>()), + _server_config(std::make_unique<StorServerConfigBuilder>()), + _status_config(std::make_unique<StorStatusConfigBuilder>()), + _visitor_config(std::make_unique<StorVisitorConfigBuilder>()), + _visitor_dispatcher_config(std::make_unique<StorVisitordispatcherConfigBuilder>()), + _config_id_str(std::move(config_id_str)), + _config_ctx(std::make_shared<config::ConfigContext>(_config_set)), + _config_uri(_config_id_str, _config_ctx) +{ + _config_set.addBuilder(_config_id_str, _document_type_config.get()); + _config_set.addBuilder(_config_id_str, _slobroks_config.get()); + _config_set.addBuilder(_config_id_str, _messagebus_config.get()); + _config_set.addBuilder(_config_id_str, _metrics_config.get()); + _config_set.addBuilder(_config_id_str, _persistence_config.get()); + _config_set.addBuilder(_config_id_str, _distribution_config.get()); + _config_set.addBuilder(_config_id_str, _filestor_config.get()); + _config_set.addBuilder(_config_id_str, _upgrading_config.get()); + _config_set.addBuilder(_config_id_str, _bucket_spaces_config.get()); + _config_set.addBuilder(_config_id_str, _bouncer_config.get()); + _config_set.addBuilder(_config_id_str, _communication_manager_config.get()); + _config_set.addBuilder(_config_id_str, _distributor_manager_config.get()); + _config_set.addBuilder(_config_id_str, _priority_mapping_config.get()); + _config_set.addBuilder(_config_id_str, _server_config.get()); + _config_set.addBuilder(_config_id_str, _status_config.get()); + _config_set.addBuilder(_config_id_str, _visitor_config.get()); + _config_set.addBuilder(_config_id_str, _visitor_dispatcher_config.get()); + + init_default_configs(is_storage_node); + _config_ctx->reload(); +} + +StorageConfigSet::~StorageConfigSet() = default; + +void StorageConfigSet::init_default_configs(bool is_storage_node) { + // Most configs are left with their default values, with explicit values being a + // union of the legacy DirConfig test helpers. + *_document_type_config = document::TestDocRepo().getTypeConfig(); + + add_metric_consumer("status", {"*"}); + add_metric_consumer("statereporter", {"*"}); + + add_distribution_config(50); + add_bucket_space_mapping("testdoctype1", "default"); + + _communication_manager_config->rpcport = 0; + _communication_manager_config->mbusport = 0; + + _distributor_manager_config->splitcount = 1000; + _distributor_manager_config->splitsize = 10000000; + _distributor_manager_config->joincount = 500; + _distributor_manager_config->joinsize = 5000000; + _distributor_manager_config->maxClusterClockSkewSec = 0; + + _filestor_config->numThreads = 1; + _filestor_config->numResponseThreads = 1; + + _persistence_config->abortOperationsWithChangedBucketOwnership = true; + + _server_config->clusterName = "storage"; + _server_config->nodeIndex = 0; + _server_config->isDistributor = !is_storage_node; + _server_config->maxMergesPerNode = 25; + _server_config->maxMergeQueueSize = 20; + _server_config->resourceExhaustionMergeBackPressureDurationSecs = 15.0; + _server_config->writePidFileOnStartup = false; + + _status_config->httpport = 0; + + _visitor_config->maxconcurrentvisitorsFixed = 4; + _visitor_config->maxconcurrentvisitorsVariable = 0; +} + +void StorageConfigSet::add_bucket_space_mapping(vespalib::string doc_type, vespalib::string bucket_space_name) { + BucketspacesConfigBuilder::Documenttype type; + type.name = std::move(doc_type); + type.bucketspace = std::move(bucket_space_name); + _bucket_spaces_config->documenttype.emplace_back(std::move(type)); +} + +void StorageConfigSet::add_distribution_config(uint16_t nodes_in_top_level_group) { + StorDistributionConfigBuilder::Group group; + group.name = "invalid"; + group.index = "invalid"; + for (uint16_t i = 0; i < nodes_in_top_level_group; ++i) { + StorDistributionConfigBuilder::Group::Nodes node; + node.index = i; + group.nodes.emplace_back(std::move(node)); + } + _distribution_config->group.clear(); + _distribution_config->group.emplace_back(std::move(group)); + _distribution_config->redundancy = 2; +} + +void StorageConfigSet::add_metric_consumer(vespalib::string name, const std::vector<vespalib::string>& added_metrics) { + MetricsmanagerConfigBuilder::Consumer consumer; + consumer.name = std::move(name); + consumer.addedmetrics.assign(added_metrics.begin(), added_metrics.end()); + _metrics_config->consumer.emplace_back(std::move(consumer)); +} + +void StorageConfigSet::set_node_index(uint16_t node_index) { + _server_config->nodeIndex = node_index; +} + +void StorageConfigSet::set_slobrok_config_port(int slobrok_port) { + SlobroksConfigBuilder::Slobrok slobrok; + slobrok.connectionspec = vespalib::make_string("tcp/localhost:%d", slobrok_port); + _slobroks_config->slobrok.clear(); + _slobroks_config->slobrok.emplace_back(std::move(slobrok)); +} + +} // storage diff --git a/storage/src/tests/common/storage_config_set.h b/storage/src/tests/common/storage_config_set.h new file mode 100644 index 00000000000..66cdeaf527f --- /dev/null +++ b/storage/src/tests/common/storage_config_set.h @@ -0,0 +1,122 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/config/common/configcontext.h> +#include <vespa/config/subscription/configuri.h> +#include <vespa/config/subscription/sourcespec.h> +#include <memory> + +// FIXME "internal" here is very counter-productive since it precludes easy fwd decls. +// Currently have to punch holes in internal abstractions to make this work at all. +namespace cloud::config::internal { class InternalSlobroksType; } +namespace messagebus::internal { class InternalMessagebusType; } +namespace metrics::internal { class InternalMetricsmanagerType; } +namespace document::config::internal { class InternalDocumenttypesType; } +namespace vespa::config::content::internal { +class InternalPersistenceType; +class InternalStorDistributionType; +class InternalStorFilestorType; +class InternalUpgradingType; +} +namespace vespa::config::content::core::internal { +class InternalBucketspacesType; +class InternalStorBouncerType; +class InternalStorCommunicationmanagerType; +class InternalStorDistributormanagerType; +class InternalStorPrioritymappingType; +class InternalStorServerType; +class InternalStorStatusType; +class InternalStorVisitorType; +class InternalStorVisitordispatcherType; +} + +namespace storage { + +class StorageConfigSet { + using SlobroksConfigBuilder = cloud::config::internal::InternalSlobroksType; + using MessagebusConfigBuilder = messagebus::internal::InternalMessagebusType; + using MetricsmanagerConfigBuilder = metrics::internal::InternalMetricsmanagerType; + using DocumenttypesConfigBuilder = document::config::internal::InternalDocumenttypesType; + using PersistenceConfigBuilder = vespa::config::content::internal::InternalPersistenceType; + using StorDistributionConfigBuilder = vespa::config::content::internal::InternalStorDistributionType; + using StorFilestorConfigBuilder = vespa::config::content::internal::InternalStorFilestorType; + using UpgradingConfigBuilder = vespa::config::content::internal::InternalUpgradingType; + using BucketspacesConfigBuilder = vespa::config::content::core::internal::InternalBucketspacesType; + using StorBouncerConfigBuilder = vespa::config::content::core::internal::InternalStorBouncerType; + using StorCommunicationmanagerConfigBuilder = vespa::config::content::core::internal::InternalStorCommunicationmanagerType; + using StorDistributormanagerConfigBuilder = vespa::config::content::core::internal::InternalStorDistributormanagerType; + using StorPrioritymappingConfigBuilder = vespa::config::content::core::internal::InternalStorPrioritymappingType; + using StorServerConfigBuilder = vespa::config::content::core::internal::InternalStorServerType; + using StorStatusConfigBuilder = vespa::config::content::core::internal::InternalStorStatusType; + using StorVisitorConfigBuilder = vespa::config::content::core::internal::InternalStorVisitorType; + using StorVisitordispatcherConfigBuilder = vespa::config::content::core::internal::InternalStorVisitordispatcherType; + + std::unique_ptr<DocumenttypesConfigBuilder> _document_type_config; + std::unique_ptr<SlobroksConfigBuilder> _slobroks_config; + std::unique_ptr<MessagebusConfigBuilder> _messagebus_config; + std::unique_ptr<MetricsmanagerConfigBuilder> _metrics_config; + std::unique_ptr<PersistenceConfigBuilder> _persistence_config; + std::unique_ptr<StorDistributionConfigBuilder> _distribution_config; + std::unique_ptr<StorFilestorConfigBuilder> _filestor_config; + std::unique_ptr<UpgradingConfigBuilder> _upgrading_config; + std::unique_ptr<BucketspacesConfigBuilder> _bucket_spaces_config; + std::unique_ptr<StorBouncerConfigBuilder> _bouncer_config; + std::unique_ptr<StorCommunicationmanagerConfigBuilder> _communication_manager_config; + std::unique_ptr<StorDistributormanagerConfigBuilder> _distributor_manager_config; + std::unique_ptr<StorPrioritymappingConfigBuilder> _priority_mapping_config; // TODO removable? + std::unique_ptr<StorServerConfigBuilder> _server_config; + std::unique_ptr<StorStatusConfigBuilder> _status_config; + std::unique_ptr<StorVisitorConfigBuilder> _visitor_config; + std::unique_ptr<StorVisitordispatcherConfigBuilder> _visitor_dispatcher_config; + + vespalib::string _config_id_str; + config::ConfigSet _config_set; + std::shared_ptr<config::ConfigContext> _config_ctx; + config::ConfigUri _config_uri; + +public: + StorageConfigSet(vespalib::string config_id_str, bool is_storage_node); + ~StorageConfigSet(); + + void init_default_configs(bool is_storage_node); + void add_bucket_space_mapping(vespalib::string doc_type, vespalib::string bucket_space_name); + void add_metric_consumer(vespalib::string name, const std::vector<vespalib::string>& added_metrics); + void add_distribution_config(uint16_t nodes_in_top_level_group); + void set_slobrok_config_port(int slobrok_port); + void set_node_index(uint16_t node_index); + + [[nodiscard]] const config::ConfigUri& config_uri() const noexcept { + return _config_uri; + } + + DocumenttypesConfigBuilder& document_type_config() noexcept { return *_document_type_config; } + SlobroksConfigBuilder& slobroks_config() noexcept { return *_slobroks_config; } + MessagebusConfigBuilder& messagebus_config() noexcept {return *_messagebus_config; } + MetricsmanagerConfigBuilder& metrics_config() noexcept { return *_metrics_config; } + PersistenceConfigBuilder& persistence_config() noexcept { return *_persistence_config; } + StorDistributionConfigBuilder& distribution_config() noexcept { return *_distribution_config; } + StorFilestorConfigBuilder& filestor_config() noexcept { return *_filestor_config; } + BucketspacesConfigBuilder& bucket_spaces_config() noexcept { return *_bucket_spaces_config; } + StorBouncerConfigBuilder& bouncer_config() noexcept { return *_bouncer_config; }; + StorCommunicationmanagerConfigBuilder& communication_manager_config() noexcept { return *_communication_manager_config; } + StorDistributormanagerConfigBuilder& distributor_manager_config() noexcept { return *_distributor_manager_config; } + StorServerConfigBuilder& server_config() noexcept { return *_server_config; } + StorStatusConfigBuilder& status_config() noexcept { return *_status_config; } + StorVisitorConfigBuilder& visitor_config() noexcept { return *_visitor_config; } + StorVisitordispatcherConfigBuilder& visitor_dispatcher_config() noexcept { return *_visitor_dispatcher_config; } + + [[nodiscard]] static std::unique_ptr<StorageConfigSet> make_node_config(bool is_storage_node) { + return std::make_unique<StorageConfigSet>("my-node", is_storage_node); + } + + [[nodiscard]] static std::unique_ptr<StorageConfigSet> make_storage_node_config() { + return make_node_config(true); + } + + [[nodiscard]] static std::unique_ptr<StorageConfigSet> make_distributor_node_config() { + return make_node_config(false); + } +}; + +} diff --git a/storage/src/tests/common/teststorageapp.cpp b/storage/src/tests/common/teststorageapp.cpp index 9e421051184..d811f100aec 100644 --- a/storage/src/tests/common/teststorageapp.cpp +++ b/storage/src/tests/common/teststorageapp.cpp @@ -26,12 +26,12 @@ namespace storage { TestStorageApp::TestStorageApp(StorageComponentRegisterImpl::UP compReg, const lib::NodeType& type, NodeIndex index, - vespalib::stringref configId) + const config::ConfigUri& config_uri) : TestComponentRegister(ComponentRegisterImpl::UP(std::move(compReg))), _compReg(dynamic_cast<StorageComponentRegisterImpl&>(TestComponentRegister::getComponentRegister())), _docMan(), _nodeStateUpdater(type), - _configId(configId), + _configId(config_uri.getConfigId()), _node_identity("test_cluster", type, index), _initialized(false) { @@ -39,17 +39,18 @@ TestStorageApp::TestStorageApp(StorageComponentRegisterImpl::UP compReg, vespalib::string clusterName = "mycluster"; uint32_t redundancy = 2; uint32_t nodeCount = 10; - if (!configId.empty()) { - config::ConfigUri uri(configId); - auto serverConfig = config::ConfigGetter<vespa::config::content::core::StorServerConfig>::getConfig(uri.getConfigId(), uri.getContext()); - clusterName = serverConfig->clusterName; - if (index == 0xffff) index = serverConfig->nodeIndex; - redundancy = config::ConfigGetter<vespa::config::content::StorDistributionConfig>::getConfig(uri.getConfigId(), uri.getContext())->redundancy; - } else { - if (index == 0xffff) index = 0; + auto serverConfig = config::ConfigGetter<vespa::config::content::core::StorServerConfig>::getConfig(config_uri.getConfigId(), config_uri.getContext()); + clusterName = serverConfig->clusterName; + if (index == 0xffff) { + index = serverConfig->nodeIndex; + } + redundancy = config::ConfigGetter<vespa::config::content::StorDistributionConfig>::getConfig(config_uri.getConfigId(), config_uri.getContext())->redundancy; + if (index >= nodeCount) { + nodeCount = index + 1; + } + if (redundancy > nodeCount) { + redundancy = nodeCount; } - if (index >= nodeCount) nodeCount = index + 1; - if (redundancy > nodeCount) redundancy = nodeCount; _compReg.setNodeInfo(clusterName, type, index); _compReg.setNodeStateUpdater(_nodeStateUpdater); @@ -83,21 +84,17 @@ TestStorageApp::setClusterState(const lib::ClusterState& c) } namespace { -NodeIndex getIndexFromConfig(vespalib::stringref configId) { - if (!configId.empty()) { - config::ConfigUri uri(configId); - return NodeIndex( - config::ConfigGetter<vespa::config::content::core::StorServerConfig>::getConfig(uri.getConfigId(), uri.getContext())->nodeIndex); - } - return NodeIndex(0); + +NodeIndex node_index_from_config(const config::ConfigUri& uri) { + return NodeIndex(config::ConfigGetter<vespa::config::content::core::StorServerConfig>::getConfig(uri.getConfigId(), uri.getContext())->nodeIndex); } VESPA_THREAD_STACK_TAG(test_executor) } -TestServiceLayerApp::TestServiceLayerApp(vespalib::stringref configId) +TestServiceLayerApp::TestServiceLayerApp(NodeIndex index, const config::ConfigUri& config_uri) : TestStorageApp(std::make_unique<ServiceLayerComponentRegisterImpl>(ContentBucketDbOptions()), - lib::NodeType::STORAGE, getIndexFromConfig(configId), configId), + lib::NodeType::STORAGE, index, config_uri), _compReg(dynamic_cast<ServiceLayerComponentRegisterImpl&>(TestStorageApp::getComponentRegister())), _persistenceProvider(), _executor(vespalib::SequencedTaskExecutor::create(test_executor, 1)), @@ -107,17 +104,9 @@ TestServiceLayerApp::TestServiceLayerApp(vespalib::stringref configId) _nodeStateUpdater.setReportedNodeState(ns); } -TestServiceLayerApp::TestServiceLayerApp(NodeIndex index, - vespalib::stringref configId) - : TestStorageApp(std::make_unique<ServiceLayerComponentRegisterImpl>(ContentBucketDbOptions()), - lib::NodeType::STORAGE, index, configId), - _compReg(dynamic_cast<ServiceLayerComponentRegisterImpl&>(TestStorageApp::getComponentRegister())), - _persistenceProvider(), - _executor(vespalib::SequencedTaskExecutor::create(test_executor, 1)), - _host_info() +TestServiceLayerApp::TestServiceLayerApp(const config::ConfigUri& config_uri) + : TestServiceLayerApp(node_index_from_config(config_uri), config_uri) { - lib::NodeState ns(*_nodeStateUpdater.getReportedNodeState()); - _nodeStateUpdater.setReportedNodeState(ns); } TestServiceLayerApp::~TestServiceLayerApp() = default; @@ -146,45 +135,37 @@ TestServiceLayerApp::getPersistenceProvider() } namespace { - template<typename T> - T getConfig(vespalib::stringref configId) { - config::ConfigUri uri(configId); - return *config::ConfigGetter<T>::getConfig(uri.getConfigId(), uri.getContext()); - } + +template<typename T> +[[nodiscard]] T get_config(const config::ConfigUri& uri) { + return *config::ConfigGetter<T>::getConfig(uri.getConfigId(), uri.getContext()); +} + } void -TestDistributorApp::configure(vespalib::stringref id) +TestDistributorApp::configure(const config::ConfigUri& config_uri) { - if (id.empty()) return; - auto dc(getConfig<vespa::config::content::core::StorDistributormanagerConfig>(id)); + auto dc = get_config<vespa::config::content::core::StorDistributormanagerConfig>(config_uri); _compReg.setDistributorConfig(dc); - auto vc(getConfig<vespa::config::content::core::StorVisitordispatcherConfig>(id)); + auto vc = get_config<vespa::config::content::core::StorVisitordispatcherConfig>(config_uri); _compReg.setVisitorConfig(vc); } -TestDistributorApp::TestDistributorApp(vespalib::stringref configId) - : TestStorageApp( - std::make_unique<DistributorComponentRegisterImpl>(), - lib::NodeType::DISTRIBUTOR, getIndexFromConfig(configId), configId), +TestDistributorApp::TestDistributorApp(NodeIndex index, const config::ConfigUri& config_uri) + : TestStorageApp(std::make_unique<DistributorComponentRegisterImpl>(), + lib::NodeType::DISTRIBUTOR, index, config_uri), _compReg(dynamic_cast<DistributorComponentRegisterImpl&>(TestStorageApp::getComponentRegister())), _lastUniqueTimestampRequested(0), _uniqueTimestampCounter(0) { _compReg.setTimeCalculator(*this); - configure(configId); + configure(config_uri); } -TestDistributorApp::TestDistributorApp(NodeIndex index, vespalib::stringref configId) - : TestStorageApp( - std::make_unique<DistributorComponentRegisterImpl>(), - lib::NodeType::DISTRIBUTOR, index, configId), - _compReg(dynamic_cast<DistributorComponentRegisterImpl&>(TestStorageApp::getComponentRegister())), - _lastUniqueTimestampRequested(0), - _uniqueTimestampCounter(0) +TestDistributorApp::TestDistributorApp(const config::ConfigUri& config_uri) + : TestDistributorApp(node_index_from_config(config_uri), config_uri) { - _compReg.setTimeCalculator(*this); - configure(configId); } TestDistributorApp::~TestDistributorApp() = default; diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index fb91145c66a..04fa6996e15 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -1,9 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * \class storage::TestServiceLayerApp - * \ingroup common - * - * \brief Helper class for tests involving service layer. + * Helper class for tests involving service layer. * * Some components need some dependencies injected in order to work correctly. * This test class simplifies the process of creating these dependencies. @@ -34,6 +31,8 @@ #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <atomic> +namespace config { class ConfigUri; } + namespace storage { namespace spi { struct PersistenceProvider; } @@ -66,8 +65,8 @@ public: * from config themselves. */ TestStorageApp(StorageComponentRegisterImpl::UP compReg, - const lib::NodeType&, NodeIndex = NodeIndex(0xffff), - vespalib::stringref configId = ""); + const lib::NodeType&, NodeIndex index, + const config::ConfigUri& config_uri); ~TestStorageApp() override; // Set functions, to be able to modify content while running. @@ -110,8 +109,8 @@ class TestServiceLayerApp : public TestStorageApp HostInfo _host_info; public: - explicit TestServiceLayerApp(vespalib::stringref configId); - explicit TestServiceLayerApp(NodeIndex = NodeIndex(0xffff), vespalib::stringref configId = ""); + TestServiceLayerApp(NodeIndex node_index, const config::ConfigUri& config_uri); + explicit TestServiceLayerApp(const config::ConfigUri& config_uri); ~TestServiceLayerApp() override; void setupDummyPersistence(); @@ -140,11 +139,11 @@ class TestDistributorApp : public TestStorageApp, uint64_t _lastUniqueTimestampRequested; uint32_t _uniqueTimestampCounter; - void configure(vespalib::stringref configId); + void configure(const config::ConfigUri& config_uri); public: - explicit TestDistributorApp(vespalib::stringref configId = ""); - explicit TestDistributorApp(NodeIndex index, vespalib::stringref configId = ""); + TestDistributorApp(NodeIndex index, const config::ConfigUri& config_uri); + explicit TestDistributorApp(const config::ConfigUri& config_uri); ~TestDistributorApp() override; DistributorComponentRegisterImpl& getComponentRegister() override { diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.cpp b/storage/src/tests/distributor/distributor_stripe_test_util.cpp index 923c7d1730b..f0b3db3adf8 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.cpp +++ b/storage/src/tests/distributor/distributor_stripe_test_util.cpp @@ -32,7 +32,7 @@ DistributorStripeTestUtil::DistributorStripeTestUtil() _done_initializing(true), _messageSender(_sender, _senderDown) { - _config = getStandardConfig(false); + _config = StorageConfigSet::make_distributor_node_config(); } DistributorStripeTestUtil::~DistributorStripeTestUtil() = default; @@ -40,7 +40,7 @@ DistributorStripeTestUtil::~DistributorStripeTestUtil() = default; void DistributorStripeTestUtil::createLinks() { - _node = std::make_unique<TestDistributorApp>(_config.getConfigId()); + _node = std::make_unique<TestDistributorApp>(_config->config_uri()); _metrics = std::make_shared<DistributorMetricSet>(); _ideal_state_metrics = std::make_shared<IdealStateMetricSet>(); _stripe = std::make_unique<DistributorStripe>(_node->getComponentRegister(), *_metrics, *_ideal_state_metrics, @@ -184,8 +184,8 @@ DistributorStripeTestUtil::close() { _stripe->flush_and_close(); _sender.clear(); - _node.reset(0); - _config = getStandardConfig(false); + _node.reset(); + _config = StorageConfigSet::make_distributor_node_config(); } namespace { diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.h b/storage/src/tests/distributor/distributor_stripe_test_util.h index 801320e2bf8..862d9bfbfba 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.h +++ b/storage/src/tests/distributor/distributor_stripe_test_util.h @@ -5,7 +5,9 @@ #include <tests/common/dummystoragelink.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> +#include <tests/common/storage_config_set.h> #include <vespa/storage/common/hostreporter/hostinfo.h> +#include <vespa/storage/config/config-stor-distributormanager.h> #include <vespa/storage/distributor/stripe_host_info_notifier.h> #include <vespa/storage/storageutil/utils.h> @@ -132,8 +134,8 @@ public: const DistributorConfiguration& getConfig(); - vdstestlib::DirConfig& getDirConfig() { - return _config; + vespa::config::content::core::StorDistributormanagerConfigBuilder& backing_config() noexcept { + return _config->distributor_manager_config(); } // TODO explicit notion of bucket spaces for tests @@ -237,7 +239,7 @@ public: void tag_content_node_supports_condition_probing(uint16_t index, bool supported); protected: - vdstestlib::DirConfig _config; + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestDistributorApp> _node; std::shared_ptr<DistributorMetricSet> _metrics; std::shared_ptr<IdealStateMetricSet> _ideal_state_metrics; diff --git a/storage/src/tests/distributor/externaloperationhandlertest.cpp b/storage/src/tests/distributor/externaloperationhandlertest.cpp index 634e4993d53..33da4727017 100644 --- a/storage/src/tests/distributor/externaloperationhandlertest.cpp +++ b/storage/src/tests/distributor/externaloperationhandlertest.cpp @@ -94,7 +94,7 @@ struct ExternalOperationHandlerTest : Test, DistributorStripeTestUtil { TEST_F(ExternalOperationHandlerTest, bucket_split_mask) { { createLinks(); - getDirConfig().getConfig("stor-distributormanager").set("minsplitcount", "16"); + backing_config().minsplitcount = 16; EXPECT_EQ(document::BucketId(16, 0xffff), operation_context().make_split_bit_constrained_bucket_id(document::DocumentId( @@ -115,7 +115,7 @@ TEST_F(ExternalOperationHandlerTest, bucket_split_mask) { close(); } { - getDirConfig().getConfig("stor-distributormanager").set("minsplitcount", "20"); + backing_config().minsplitcount = 20; createLinks(); EXPECT_EQ(document::BucketId(20, 0x11111), operation_context().make_split_bit_constrained_bucket_id(document::DocumentId( diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp index 0cadaa3fc9f..4639a154e74 100644 --- a/storage/src/tests/distributor/idealstatemanagertest.cpp +++ b/storage/src/tests/distributor/idealstatemanagertest.cpp @@ -74,10 +74,10 @@ TEST_F(IdealStateManagerTest, sibling) { TEST_F(IdealStateManagerTest, status_page) { close(); - getDirConfig().getConfig("stor-distributormanager").set("splitsize", "100"); - getDirConfig().getConfig("stor-distributormanager").set("splitcount", "1000000"); - getDirConfig().getConfig("stor-distributormanager").set("joinsize", "0"); - getDirConfig().getConfig("stor-distributormanager").set("joincount", "0"); + backing_config().splitsize = 100; + backing_config().splitcount = 1000000; + backing_config().joinsize = 0; + backing_config().joincount = 0; createLinks(); setup_stripe(1, 1, "distributor:1 storage:1"); diff --git a/storage/src/tests/distributor/statusreporterdelegatetest.cpp b/storage/src/tests/distributor/statusreporterdelegatetest.cpp index cc23fa7a22e..c70ab533af6 100644 --- a/storage/src/tests/distributor/statusreporterdelegatetest.cpp +++ b/storage/src/tests/distributor/statusreporterdelegatetest.cpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <vespa/storage/distributor/statusreporterdelegate.h> @@ -45,8 +46,8 @@ public: } TEST(StatusReporterDelegateTest, delegate_invokes_delegator_on_status_request) { - vdstestlib::DirConfig config(getStandardConfig(false)); - TestDistributorApp app(config.getConfigId()); + auto config = StorageConfigSet::make_distributor_node_config(); + TestDistributorApp app(config->config_uri()); MockDelegator mockDelegator; MockStatusReporter reporter; diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.cpp b/storage/src/tests/distributor/top_level_distributor_test_util.cpp index 94031c6d71e..e1b2bc93f62 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.cpp +++ b/storage/src/tests/distributor/top_level_distributor_test_util.cpp @@ -24,7 +24,7 @@ TopLevelDistributorTestUtil::TopLevelDistributorTestUtil() : _message_sender(_sender, _sender_down), _num_distributor_stripes(4) { - _config = getStandardConfig(false); + _config = StorageConfigSet::make_distributor_node_config(); } TopLevelDistributorTestUtil::~TopLevelDistributorTestUtil() = default; @@ -32,7 +32,7 @@ TopLevelDistributorTestUtil::~TopLevelDistributorTestUtil() = default; void TopLevelDistributorTestUtil::create_links() { - _node = std::make_unique<TestDistributorApp>(_config.getConfigId()); + _node = std::make_unique<TestDistributorApp>(_config->config_uri()); _thread_pool = framework::TickingThreadPool::createDefault("distributor", 100ms); _stripe_pool = DistributorStripePool::make_non_threaded_pool_for_testing(); _distributor.reset(new TopLevelDistributor( @@ -123,7 +123,7 @@ TopLevelDistributorTestUtil::close() } _sender.clear(); _node.reset(); - _config = getStandardConfig(false); + _config = StorageConfigSet::make_distributor_node_config(); } void diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.h b/storage/src/tests/distributor/top_level_distributor_test_util.h index 1d4c81a5bfb..51f0739e3e6 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.h +++ b/storage/src/tests/distributor/top_level_distributor_test_util.h @@ -3,6 +3,7 @@ #include "distributor_message_sender_stub.h" #include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <vespa/storage/common/hostreporter/hostinfo.h> @@ -140,7 +141,7 @@ public: static std::vector<document::BucketSpace> bucket_spaces(); protected: - vdstestlib::DirConfig _config; + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestDistributorApp> _node; std::unique_ptr<framework::TickingThreadPool> _thread_pool; std::unique_ptr<DistributorStripePool> _stripe_pool; diff --git a/storage/src/tests/frameworkimpl/status/statustest.cpp b/storage/src/tests/frameworkimpl/status/statustest.cpp index bd28297e108..8592a332f0c 100644 --- a/storage/src/tests/frameworkimpl/status/statustest.cpp +++ b/storage/src/tests/frameworkimpl/status/statustest.cpp @@ -1,10 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/storage_config_set.h> +#include <tests/common/teststorageapp.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> #include <vespa/storage/frameworkimpl/status/statuswebserver.h> #include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/storageframework/generic/status/xmlstatusreporter.h> -#include <tests/common/teststorageapp.h> #include <vespa/document/util/stringutil.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/socket_spec.h> @@ -39,6 +40,7 @@ vespalib::string fetch(int port, const vespalib::string &path) { namespace storage { struct StatusTest : Test { + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestServiceLayerApp> _node; void SetUp() override; @@ -97,7 +99,8 @@ namespace { } void StatusTest::SetUp() { - _node = std::make_unique<TestServiceLayerApp>(); + _config = StorageConfigSet::make_storage_node_config(); + _node = std::make_unique<TestServiceLayerApp>(_config->config_uri()); } namespace { diff --git a/storage/src/tests/persistence/bucketownershipnotifiertest.cpp b/storage/src/tests/persistence/bucketownershipnotifiertest.cpp index 129cac34c68..717a79b030d 100644 --- a/storage/src/tests/persistence/bucketownershipnotifiertest.cpp +++ b/storage/src/tests/persistence/bucketownershipnotifiertest.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/message_sender_stub.h> +#include <tests/common/storage_config_set.h> #include <tests/common/teststorageapp.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/persistence/bucketownershipnotifier.h> @@ -14,6 +15,7 @@ using namespace ::testing; namespace storage { struct BucketOwnershipNotifierTest : public Test { + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestServiceLayerApp> _app; lib::ClusterState _clusterState; @@ -58,7 +60,8 @@ struct BucketOwnershipNotifierTest : public Test { void BucketOwnershipNotifierTest::SetUp() { - _app = std::make_unique<TestServiceLayerApp>(); + _config = StorageConfigSet::make_storage_node_config(); + _app = std::make_unique<TestServiceLayerApp>(_config->config_uri()); _app->setDistribution(Redundancy(1), NodeCount(2)); _app->setClusterState(_clusterState); } diff --git a/storage/src/tests/persistence/common/filestortestfixture.cpp b/storage/src/tests/persistence/common/filestortestfixture.cpp index 6581fb9f7b1..6c7b09c4736 100644 --- a/storage/src/tests/persistence/common/filestortestfixture.cpp +++ b/storage/src/tests/persistence/common/filestortestfixture.cpp @@ -25,14 +25,11 @@ const uint32_t FileStorTestFixture::MSG_WAIT_TIME; void FileStorTestFixture::setupPersistenceThreads(uint32_t threads) { - std::string rootOfRoot = "todo-make-unique-filestorefixture"; - _config = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true, rootOfRoot)); - _config->getConfig("stor-server").set("root_folder", (rootOfRoot + "-vdsroot.2")); - _config->getConfig("stor-devices").set("root_folder", (rootOfRoot + "-vdsroot.2")); - _config->getConfig("stor-server").set("node_index", "1"); - _config->getConfig("stor-filestor").set("num_threads", std::to_string(threads)); - - _node = std::make_unique<TestServiceLayerApp>(NodeIndex(1), _config->getConfigId()); + _config = StorageConfigSet::make_storage_node_config(); + _config->set_node_index(1); + _config->filestor_config().numThreads = threads; + + _node = std::make_unique<TestServiceLayerApp>(NodeIndex(1), _config->config_uri()); _testdoctype1 = _node->getTypeRepo()->getDocumentType("testdoctype1"); } @@ -77,7 +74,7 @@ FileStorTestFixture::TestFileStorComponents::TestFileStorComponents( { injector.inject(top); using StorFilestorConfig = vespa::config::content::internal::InternalStorFilestorType; - auto config = config_from<StorFilestorConfig>(config::ConfigUri(fixture._config->getConfigId())); + auto config = config_from<StorFilestorConfig>(fixture._config->config_uri()); auto fsm = std::make_unique<FileStorManager>(*config, fixture._node->getPersistenceProvider(), fixture._node->getComponentRegister(), *fixture._node, fixture._node->get_host_info()); manager = fsm.get(); diff --git a/storage/src/tests/persistence/common/filestortestfixture.h b/storage/src/tests/persistence/common/filestortestfixture.h index e4776f393ae..d4fb94101cc 100644 --- a/storage/src/tests/persistence/common/filestortestfixture.h +++ b/storage/src/tests/persistence/common/filestortestfixture.h @@ -2,6 +2,7 @@ #pragma once #include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <vespa/storage/persistence/filestorage/filestorhandlerimpl.h> @@ -14,8 +15,8 @@ namespace storage { class FileStorTestFixture : public ::testing::Test { public: + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestServiceLayerApp> _node; - std::unique_ptr<vdstestlib::DirConfig> _config; const document::DocumentType* _testdoctype1; static const uint32_t MSG_WAIT_TIME = 60 * 1000; diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 2680eacf49c..bdc3f17e576 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <tests/persistence/filestorage/forwardingmessagesender.h> @@ -41,7 +42,6 @@ #include <vespa/log/log.h> LOG_SETUP(".filestormanagertest"); -using std::unique_ptr; using document::Document; using document::BucketId; using namespace storage::api; @@ -91,8 +91,8 @@ make_bucket_for_doc(const document::DocumentId& docid) struct FileStorTestBase : Test { enum {LONG_WAITTIME=60}; - unique_ptr<TestServiceLayerApp> _node; - std::unique_ptr<vdstestlib::DirConfig> config; + std::unique_ptr<StorageConfigSet> _config; + std::unique_ptr<TestServiceLayerApp> _node; const int32_t _waitTime; const document::DocumentType* _testdoctype1; @@ -163,9 +163,9 @@ struct FileStorTestBase : Test { void setupDisks() { std::string rootOfRoot = "filestormanagertest"; - config = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true, rootOfRoot)); + _config = StorageConfigSet::make_storage_node_config(); - _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), config->getConfigId()); + _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), _config->config_uri()); _node->setupDummyPersistence(); _testdoctype1 = _node->getTypeRepo()->getDocumentType("testdoctype1"); } @@ -209,8 +209,7 @@ struct TestFileStorComponents { explicit TestFileStorComponents(FileStorTestBase& test) : manager(nullptr) { - auto config_uri = config::ConfigUri(test.config->getConfigId()); - auto config = config_from<StorFilestorConfig>(config_uri); + auto config = config_from<StorFilestorConfig>(test._config->config_uri()); auto fsm = std::make_unique<FileStorManager>(*config, test._node->getPersistenceProvider(), test._node->getComponentRegister(), *test._node, test._node->get_host_info()); manager = fsm.get(); @@ -1388,8 +1387,7 @@ TEST_F(FileStorManagerTest, remove_location) { TEST_F(FileStorManagerTest, delete_bucket) { TestFileStorComponents c(*this); - auto config_uri = config::ConfigUri(config->getConfigId()); - StorFilestorConfigBuilder my_config(*config_from<StorFilestorConfig>(config_uri)); + auto my_config = *config_from<StorFilestorConfig>(_config->config_uri()); c.manager->on_configure(my_config); auto& top = c.top; diff --git a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp index 710da80972f..38acc4a18b8 100644 --- a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp @@ -38,7 +38,7 @@ struct BucketCheckerInjector : FileStorTestFixture::StorageLinkInjector {} void inject(DummyStorageLink& link) const override { using vespa::config::content::core::StorServerConfig; - auto cfg = config_from<StorServerConfig>(config::ConfigUri(_fixture._config->getConfigId())); + auto cfg = config_from<StorServerConfig>(_fixture._config->config_uri()); link.push_back(std::make_unique<ModifiedBucketChecker>( _node.getComponentRegister(), _node.getPersistenceProvider(), *cfg)); } diff --git a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp index f96ff9c012e..9fc6ddff268 100644 --- a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp +++ b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <vespa/config/common/exceptions.h> @@ -34,20 +35,20 @@ struct ModifiedBucketCheckerTest : Test { ModifiedBucketChecker* _handler; DummyStorageLink* _bottom; + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestServiceLayerApp> _node; - std::unique_ptr<vdstestlib::DirConfig> _config; }; void ModifiedBucketCheckerTest::SetUp() { - _config.reset(new vdstestlib::DirConfig(getStandardConfig(true))); - _node.reset(new TestServiceLayerApp(NodeIndex(0), _config->getConfigId())); + _config = StorageConfigSet::make_storage_node_config(); + _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), _config->config_uri()); _node->setupDummyPersistence(); - _top.reset(new DummyStorageLink); + _top = std::make_unique<DummyStorageLink>(); using vespa::config::content::core::StorServerConfig; - auto bootstrap_cfg = config_from<StorServerConfig>(config::ConfigUri(_config->getConfigId())); + auto bootstrap_cfg = config_from<StorServerConfig>(_config->config_uri()); _handler = new ModifiedBucketChecker(_node->getComponentRegister(), _node->getPersistenceProvider(), *bootstrap_cfg); diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index 4a2ee987362..b63df53781d 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -25,10 +25,6 @@ namespace storage { namespace { -vdstestlib::DirConfig initialize(const std::string & rootOfRoot) { - return getStandardConfig(true, rootOfRoot); -} - template<typename T> struct ConfigReader : public T::Subscriber { @@ -45,10 +41,10 @@ constexpr uint32_t MERGE_CHUNK_SIZE = 4_Mi; } -PersistenceTestEnvironment::PersistenceTestEnvironment(const std::string & rootOfRoot) - : _config(initialize(rootOfRoot)), +PersistenceTestEnvironment::PersistenceTestEnvironment() + : _config(StorageConfigSet::make_distributor_node_config()), _messageKeeper(), - _node(NodeIndex(0), _config.getConfigId()), + _node(NodeIndex(0), _config->config_uri()), _component(_node.getComponentRegister(), "persistence test env"), _metrics() { @@ -102,7 +98,7 @@ PersistenceTestUtils::MockBucketLocks::unlock(document::Bucket bucket) } PersistenceTestUtils::PersistenceTestUtils() - : _env(std::make_unique<PersistenceTestEnvironment>("todo-make-unique-persistencetestutils")), + : _env(std::make_unique<PersistenceTestEnvironment>()), _replySender(), _bucketOwnershipNotifier(getEnv()._component, getEnv()._fileStorHandler), _mock_bucket_locks(), diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h index d03974855ad..0125bd7aa79 100644 --- a/storage/src/tests/persistence/persistencetestutils.h +++ b/storage/src/tests/persistence/persistencetestutils.h @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <tests/common/storage_config_set.h> #include <tests/common/teststorageapp.h> #include <tests/common/testhelper.h> #include <vespa/storage/persistence/persistencethread.h> @@ -25,11 +26,11 @@ struct MessageKeeper : public MessageSender { }; struct PersistenceTestEnvironment { - PersistenceTestEnvironment(const std::string & rootOfRoot); + PersistenceTestEnvironment(); ~PersistenceTestEnvironment(); document::TestDocMan _testDocMan; - vdstestlib::DirConfig _config; + std::unique_ptr<StorageConfigSet> _config; MessageKeeper _messageKeeper; TestServiceLayerApp _node; ServiceLayerComponent _component; diff --git a/storage/src/tests/persistence/provider_error_wrapper_test.cpp b/storage/src/tests/persistence/provider_error_wrapper_test.cpp index fc88428f915..d5ce8400b25 100644 --- a/storage/src/tests/persistence/provider_error_wrapper_test.cpp +++ b/storage/src/tests/persistence/provider_error_wrapper_test.cpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/storage_config_set.h> #include <vespa/persistence/spi/test.h> #include <tests/persistence/persistencetestutils.h> #include <tests/persistence/common/persistenceproviderwrapper.h> @@ -33,13 +34,15 @@ struct MockErrorListener : ProviderErrorListener { struct Fixture { // We wrap the wrapper. It's turtles all the way down! PersistenceProviderWrapper providerWrapper; + std::unique_ptr<StorageConfigSet> config; TestServiceLayerApp app; ServiceLayerComponent component; ProviderErrorWrapper errorWrapper; Fixture(spi::PersistenceProvider& provider) : providerWrapper(provider), - app(), + config(StorageConfigSet::make_storage_node_config()), + app(config->config_uri()), component(app.getComponentRegister(), "dummy"), errorWrapper(providerWrapper) { diff --git a/storage/src/tests/storageserver/bouncertest.cpp b/storage/src/tests/storageserver/bouncertest.cpp index 296ed6d23bc..11742dd658f 100644 --- a/storage/src/tests/storageserver/bouncertest.cpp +++ b/storage/src/tests/storageserver/bouncertest.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <vespa/config/common/exceptions.h> @@ -26,6 +27,7 @@ using namespace ::testing; namespace storage { struct BouncerTest : public Test { + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestStorageApp> _node; std::unique_ptr<DummyStorageLink> _upper; Bouncer* _manager; @@ -57,15 +59,15 @@ BouncerTest::BouncerTest() } void BouncerTest::setUpAsNode(const lib::NodeType& type) { - vdstestlib::DirConfig config(getStandardConfig(type == lib::NodeType::STORAGE)); + _config = StorageConfigSet::make_node_config(type == lib::NodeType::STORAGE); if (type == lib::NodeType::STORAGE) { - _node = std::make_unique<TestServiceLayerApp>(NodeIndex(2), config.getConfigId()); + _node = std::make_unique<TestServiceLayerApp>(NodeIndex(2), _config->config_uri()); } else { - _node = std::make_unique<TestDistributorApp>(NodeIndex(2), config.getConfigId()); + _node = std::make_unique<TestDistributorApp>(NodeIndex(2), _config->config_uri()); } _upper = std::make_unique<DummyStorageLink>(); using StorBouncerConfig = vespa::config::content::core::StorBouncerConfig; - auto cfg_uri = config::ConfigUri(config.getConfigId()); + auto& cfg_uri = _config->config_uri(); auto cfg = config::ConfigGetter<StorBouncerConfig>::getConfig(cfg_uri.getConfigId(), cfg_uri.getContext()); _manager = new Bouncer(_node->getComponentRegister(), *cfg); _lower = new DummyStorageLink(); diff --git a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp index 50977b5ec8b..8982b02f2b7 100644 --- a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp +++ b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp @@ -3,6 +3,7 @@ #include <tests/common/teststorageapp.h> #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/document/base/testdocman.h> #include <vespa/storage/bucketdb/storbucketdb.h> @@ -28,11 +29,12 @@ using namespace ::testing; namespace storage { struct ChangedBucketOwnershipHandlerTest : Test { + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestServiceLayerApp> _app; - std::unique_ptr<DummyStorageLink> _top; - ChangedBucketOwnershipHandler* _handler; - DummyStorageLink* _bottom; - document::TestDocMan _testDocRepo; + std::unique_ptr<DummyStorageLink> _top; + ChangedBucketOwnershipHandler* _handler; + DummyStorageLink* _bottom; + document::TestDocMan _testDocRepo; // TODO test: down edge triggered on cluster state with cluster down? @@ -126,11 +128,12 @@ void ChangedBucketOwnershipHandlerTest::SetUp() { using vespa::config::content::PersistenceConfig; - vdstestlib::DirConfig config(getStandardConfig(true)); - _app.reset(new TestServiceLayerApp); - _top.reset(new DummyStorageLink); - _handler = new ChangedBucketOwnershipHandler(*config_from<PersistenceConfig>(config::ConfigUri(config.getConfigId())), + _config = StorageConfigSet::make_storage_node_config(); + _app = std::make_unique<TestServiceLayerApp>(NodeIndex(0), _config->config_uri()); + _top = std::make_unique<DummyStorageLink>(); + + _handler = new ChangedBucketOwnershipHandler(*config_from<PersistenceConfig>(_config->config_uri()), _app->getComponentRegister()); _top->push_back(std::unique_ptr<StorageLink>(_handler)); _bottom = new DummyStorageLink; diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp index 04322562d08..b741d79582f 100644 --- a/storage/src/tests/storageserver/communicationmanagertest.cpp +++ b/storage/src/tests/storageserver/communicationmanagertest.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <vespa/config/helper/configgetter.hpp> @@ -65,20 +66,20 @@ wait_for_slobrok_visibility(const CommunicationManager& mgr, TEST_F(CommunicationManagerTest, simple) { mbus::Slobrok slobrok; - vdstestlib::DirConfig distConfig(getStandardConfig(false)); - vdstestlib::DirConfig storConfig(getStandardConfig(true)); - distConfig.getConfig("stor-server").set("node_index", "1"); - storConfig.getConfig("stor-server").set("node_index", "1"); - addSlobrokConfig(distConfig, slobrok); - addSlobrokConfig(storConfig, slobrok); + auto dist_config = StorageConfigSet::make_distributor_node_config(); + auto stor_config = StorageConfigSet::make_storage_node_config(); + dist_config->set_node_index(1); + stor_config->set_node_index(1); + dist_config->set_slobrok_config_port(slobrok.port()); + stor_config->set_slobrok_config_port(slobrok.port()); + + auto& dist_cfg_uri = dist_config->config_uri(); + auto& stor_cfg_uri = stor_config->config_uri(); // Set up a "distributor" and a "storage" node with communication // managers and a dummy storage link below we can use for testing. - TestServiceLayerApp storNode(storConfig.getConfigId()); - TestDistributorApp distNode(distConfig.getConfigId()); - - auto dist_cfg_uri = config::ConfigUri(distConfig.getConfigId()); - auto stor_cfg_uri = config::ConfigUri(storConfig.getConfigId()); + TestServiceLayerApp storNode(stor_cfg_uri); + TestDistributorApp distNode(dist_cfg_uri); CommunicationManager distributor(distNode.getComponentRegister(), dist_cfg_uri, *config_from<CommunicationManagerConfig>(dist_cfg_uri)); @@ -123,23 +124,22 @@ void CommunicationManagerTest::doTestConfigPropagation(bool isContentNode) { mbus::Slobrok slobrok; - vdstestlib::DirConfig config(getStandardConfig(isContentNode)); - config.getConfig("stor-server").set("node_index", "1"); - auto& cfg = config.getConfig("stor-communicationmanager"); - cfg.set("mbus_content_node_max_pending_count", "12345"); - cfg.set("mbus_content_node_max_pending_size", "555666"); - cfg.set("mbus_distributor_node_max_pending_count", "6789"); - cfg.set("mbus_distributor_node_max_pending_size", "777888"); - addSlobrokConfig(config, slobrok); + auto config = StorageConfigSet::make_node_config(isContentNode); + config->set_node_index(1); + config->set_slobrok_config_port(slobrok.port()); + config->communication_manager_config().mbusContentNodeMaxPendingCount = 12345; + config->communication_manager_config().mbusContentNodeMaxPendingSize = 555666; + config->communication_manager_config().mbusDistributorNodeMaxPendingCount = 6789; + config->communication_manager_config().mbusDistributorNodeMaxPendingSize = 777888; + auto& cfg_uri = config->config_uri(); std::unique_ptr<TestStorageApp> node; if (isContentNode) { - node = std::make_unique<TestServiceLayerApp>(config.getConfigId()); + node = std::make_unique<TestServiceLayerApp>(cfg_uri); } else { - node = std::make_unique<TestDistributorApp>(config.getConfigId()); + node = std::make_unique<TestDistributorApp>(cfg_uri); } - auto cfg_uri = config::ConfigUri(config.getConfigId()); CommunicationManager commMgr(node->getComponentRegister(), cfg_uri, *config_from<CommunicationManagerConfig>(cfg_uri)); auto* storageLink = new DummyStorageLink(); @@ -180,12 +180,12 @@ TEST_F(CommunicationManagerTest, stor_pending_limit_configs_are_propagated_to_me TEST_F(CommunicationManagerTest, commands_are_dequeued_in_fifo_order) { mbus::Slobrok slobrok; - vdstestlib::DirConfig storConfig(getStandardConfig(true)); - storConfig.getConfig("stor-server").set("node_index", "1"); - addSlobrokConfig(storConfig, slobrok); - TestServiceLayerApp storNode(storConfig.getConfigId()); + auto config = StorageConfigSet::make_storage_node_config(); + config->set_node_index(1); + config->set_slobrok_config_port(slobrok.port()); + auto& cfg_uri = config->config_uri(); + TestServiceLayerApp storNode(cfg_uri); - auto cfg_uri = config::ConfigUri(storConfig.getConfigId()); CommunicationManager storage(storNode.getComponentRegister(), cfg_uri, *config_from<CommunicationManagerConfig>(cfg_uri)); auto* storageLink = new DummyStorageLink(); @@ -214,12 +214,12 @@ TEST_F(CommunicationManagerTest, commands_are_dequeued_in_fifo_order) { TEST_F(CommunicationManagerTest, replies_are_dequeued_in_fifo_order) { mbus::Slobrok slobrok; - vdstestlib::DirConfig storConfig(getStandardConfig(true)); - storConfig.getConfig("stor-server").set("node_index", "1"); - addSlobrokConfig(storConfig, slobrok); - TestServiceLayerApp storNode(storConfig.getConfigId()); + auto config = StorageConfigSet::make_storage_node_config(); + config->set_node_index(1); + config->set_slobrok_config_port(slobrok.port()); + auto& cfg_uri = config->config_uri(); + TestServiceLayerApp storNode(cfg_uri); - auto cfg_uri = config::ConfigUri(storConfig.getConfigId()); CommunicationManager storage(storNode.getComponentRegister(), cfg_uri, *config_from<CommunicationManagerConfig>(cfg_uri)); auto* storageLink = new DummyStorageLink(); @@ -249,19 +249,21 @@ struct MockMbusReplyHandler : mbus::IReplyHandler { }; struct CommunicationManagerFixture { + std::unique_ptr<StorageConfigSet> config; MockMbusReplyHandler reply_handler; mbus::Slobrok slobrok; std::unique_ptr<TestServiceLayerApp> node; std::unique_ptr<CommunicationManager> comm_mgr; DummyStorageLink* bottom_link; - CommunicationManagerFixture() { - vdstestlib::DirConfig stor_config(getStandardConfig(true)); - stor_config.getConfig("stor-server").set("node_index", "1"); - addSlobrokConfig(stor_config, slobrok); + CommunicationManagerFixture() + : config(StorageConfigSet::make_storage_node_config()) + { + config->set_node_index(1); + config->set_slobrok_config_port(slobrok.port()); + auto& cfg_uri = config->config_uri(); - node = std::make_unique<TestServiceLayerApp>(stor_config.getConfigId()); - auto cfg_uri = config::ConfigUri(stor_config.getConfigId()); + node = std::make_unique<TestServiceLayerApp>(cfg_uri); comm_mgr = std::make_unique<CommunicationManager>(node->getComponentRegister(), cfg_uri, *config_from<CommunicationManagerConfig>(cfg_uri)); bottom_link = new DummyStorageLink(); diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index cdf203b8a39..bc1c7f60706 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -1,10 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/document/test/make_document_bucket.h> #include <vespa/messagebus/dynamicthrottlepolicy.h> +#include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/persistence/messages.h> #include <vespa/storage/storageserver/mergethrottler.h> #include <vespa/storageapi/message/bucket.h> @@ -36,9 +38,8 @@ using StorServerConfigBuilder = vespa::config::content::core::StorServerConfigBu vespalib::string _storage("storage"); std::unique_ptr<StorServerConfig> default_server_config() { - vdstestlib::DirConfig dir_config(getStandardConfig(true)); - auto cfg_uri = ::config::ConfigUri(dir_config.getConfigId()); - return config_from<StorServerConfig>(cfg_uri); + auto config = StorageConfigSet::make_storage_node_config(); + return config_from<StorServerConfig>(config->config_uri()); } struct MergeBuilder { @@ -153,8 +154,9 @@ struct MergeThrottlerTest : Test { static constexpr int _messageWaitTime = 100; // Using n storage node links and dummy servers - std::vector<std::shared_ptr<DummyStorageLink> > _topLinks; - std::vector<std::shared_ptr<TestServiceLayerApp> > _servers; + std::unique_ptr<StorageConfigSet> _config; + std::vector<std::shared_ptr<DummyStorageLink>> _topLinks; + std::vector<std::shared_ptr<TestServiceLayerApp>> _servers; std::vector<MergeThrottler*> _throttlers; std::vector<DummyStorageLink*> _bottomLinks; @@ -198,14 +200,14 @@ MergeThrottlerTest::~MergeThrottlerTest() = default; void MergeThrottlerTest::SetUp() { - auto config = default_server_config(); + _config = StorageConfigSet::make_storage_node_config(); for (int i = 0; i < _storageNodeCount; ++i) { - auto server = std::make_unique<TestServiceLayerApp>(NodeIndex(i)); + auto server = std::make_unique<TestServiceLayerApp>(NodeIndex(i), _config->config_uri()); server->setClusterState(lib::ClusterState("distributor:100 storage:100 version:1")); std::unique_ptr<DummyStorageLink> top; top = std::make_unique<DummyStorageLink>(); - auto* throttler = new MergeThrottler(*config, server->getComponentRegister(), vespalib::HwInfo()); + auto* throttler = new MergeThrottler(_config->server_config(), server->getComponentRegister(), vespalib::HwInfo()); // MergeThrottler will be sandwiched in between two dummy links top->push_back(std::unique_ptr<StorageLink>(throttler)); auto* bottom = new DummyStorageLink; diff --git a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp index 6e9485e24d4..c3641b9bc56 100644 --- a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/storage_config_set.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/messagebus/testlib/slobrok.h> @@ -11,7 +12,6 @@ #include <vespa/storageapi/message/state.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <tests/common/testhelper.h> #include <vespa/vespalib/gtest/gtest.h> #include <vector> @@ -43,7 +43,7 @@ struct DummyReturnHandler : FRT_IReturnHandler { struct FixtureBase { mbus::Slobrok slobrok; - vdstestlib::DirConfig config; + std::unique_ptr<StorageConfigSet> config; MockOperationDispatcher dispatcher; std::unique_ptr<SharedRpcResources> shared_rpc_resources; std::unique_ptr<ClusterControllerApiRpcService> cc_service; @@ -52,12 +52,12 @@ struct FixtureBase { FRT_RPCRequest* bound_request{nullptr}; FixtureBase() - : config(getStandardConfig(true)) + : config(StorageConfigSet::make_storage_node_config()) { - config.getConfig("stor-server").set("node_index", "1"); - addSlobrokConfig(config, slobrok); + config->set_node_index(1); + config->set_slobrok_config_port(slobrok.port()); - shared_rpc_resources = std::make_unique<SharedRpcResources>(config::ConfigUri(config.getConfigId()), 0, 1, 1); + shared_rpc_resources = std::make_unique<SharedRpcResources>(config->config_uri(), 0, 1, 1); cc_service = std::make_unique<ClusterControllerApiRpcService>(dispatcher, *shared_rpc_resources); shared_rpc_resources->start_server_and_register_slobrok("my_cool_rpc_test"); } diff --git a/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp index 72ddc89f9d3..010f2b441ef 100644 --- a/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <tests/common/testhelper.h> +#include <tests/common/storage_config_set.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> @@ -102,7 +102,7 @@ vespalib::string to_slobrok_id(const api::StorageMessageAddress& address) { class RpcNode { protected: - vdstestlib::DirConfig _config; + std::unique_ptr<StorageConfigSet> _config; std::shared_ptr<const document::DocumentTypeRepo> _doc_type_repo; LockingMockOperationDispatcher _messages; std::unique_ptr<MessageCodecProvider> _codec_provider; @@ -111,17 +111,15 @@ protected: vespalib::string _slobrok_id; public: RpcNode(uint16_t node_index, bool is_distributor, const mbus::Slobrok& slobrok) - : _config(getStandardConfig(true)), + : _config(StorageConfigSet::make_node_config(!is_distributor)), _doc_type_repo(document::TestDocRepo().getTypeRepoSp()), _node_address(make_address(node_index, is_distributor)), _slobrok_id(to_slobrok_id(_node_address)) { - auto& cfg = _config.getConfig("stor-server"); - cfg.set("node_index", std::to_string(node_index)); - cfg.set("is_distributor", is_distributor ? "true" : "false"); - addSlobrokConfig(_config, slobrok); + _config->set_node_index(node_index); + _config->set_slobrok_config_port(slobrok.port()); - _shared_rpc_resources = std::make_unique<SharedRpcResources>(config::ConfigUri(_config.getConfigId()), 0, 1, 1); + _shared_rpc_resources = std::make_unique<SharedRpcResources>(_config->config_uri(), 0, 1, 1); // TODO make codec provider into interface so we can test decode-failures more easily? _codec_provider = std::make_unique<MessageCodecProvider>(_doc_type_repo); } diff --git a/storage/src/tests/storageserver/service_layer_error_listener_test.cpp b/storage/src/tests/storageserver/service_layer_error_listener_test.cpp index 63d8eec6dc3..b84f96dd847 100644 --- a/storage/src/tests/storageserver/service_layer_error_listener_test.cpp +++ b/storage/src/tests/storageserver/service_layer_error_listener_test.cpp @@ -1,12 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/storage_config_set.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/storage/storageserver/mergethrottler.h> #include <vespa/storage/storageserver/service_layer_error_listener.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> -#include <vespa/vdstestlib/config/dirconfig.h> #include <vespa/vespalib/gtest/gtest.h> using namespace ::testing; @@ -37,10 +37,10 @@ private: struct Fixture { using StorServerConfig = vespa::config::content::core::StorServerConfig; - vdstestlib::DirConfig config{getStandardConfig(true)}; - TestServiceLayerApp app; + std::unique_ptr<StorageConfigSet> config{StorageConfigSet::make_storage_node_config()}; + TestServiceLayerApp app{config->config_uri()}; ServiceLayerComponent component{app.getComponentRegister(), "dummy"}; - MergeThrottler merge_throttler{*config_from<StorServerConfig>(config::ConfigUri(config.getConfigId())), + MergeThrottler merge_throttler{*config_from<StorServerConfig>(config->config_uri()), app.getComponentRegister(), vespalib::HwInfo()}; TestShutdownListener shutdown_listener; ServiceLayerErrorListener error_listener{component, merge_throttler}; diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp index 2a5af397aca..b785bc141b6 100644 --- a/storage/src/tests/storageserver/statemanagertest.cpp +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -1,13 +1,14 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> #include <vespa/vdslib/state/cluster_state_bundle.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/storageserver/statemanager.h> -#include <tests/common/teststorageapp.h> -#include <tests/common/testhelper.h> -#include <tests/common/dummystoragelink.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> @@ -20,6 +21,7 @@ using namespace ::testing; namespace storage { struct StateManagerTest : Test, NodeStateReporter { + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<TestServiceLayerApp> _node; std::unique_ptr<DummyStorageLink> _upper; StateManager* _manager; @@ -46,7 +48,8 @@ struct StateManagerTest : Test, NodeStateReporter { }; StateManagerTest::StateManagerTest() - : _node(), + : _config(), + _node(), _upper(), _manager(nullptr), _lower(nullptr) @@ -56,7 +59,8 @@ StateManagerTest::StateManagerTest() void StateManagerTest::SetUp() { - _node = std::make_unique<TestServiceLayerApp>(NodeIndex(2)); + _config = StorageConfigSet::make_storage_node_config(); + _node = std::make_unique<TestServiceLayerApp>(NodeIndex(2), _config->config_uri()); // Clock will increase 1 sec per call. _node->getClock().setAbsoluteTimeInSeconds(1); _upper = std::make_unique<DummyStorageLink>(); diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index c233c6e9314..29d3daf9b86 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -1,14 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/persistence/filestorage/filestormetrics.h> #include <vespa/storage/storageserver/applicationgenerationfetcher.h> #include <vespa/storage/storageserver/statereporter.h> #include <vespa/metrics/metricmanager.h> -#include <tests/common/teststorageapp.h> -#include <tests/common/testhelper.h> -#include <tests/common/dummystoragelink.h> #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/simple_buffer.h> @@ -35,7 +36,7 @@ struct StateReporterTest : Test { std::unique_ptr<DummyStorageLink> _top; DummyApplicationGenerationFether _generationFetcher; std::unique_ptr<StateReporter> _stateReporter; - std::unique_ptr<vdstestlib::DirConfig> _config; + std::unique_ptr<StorageConfigSet> _config; std::unique_ptr<metrics::MetricSet> _topSet; std::unique_ptr<metrics::MetricManager> _metricManager; std::shared_ptr<FileStorMetrics> _filestorMetrics; @@ -68,9 +69,8 @@ StateReporterTest::StateReporterTest() StateReporterTest::~StateReporterTest() = default; void StateReporterTest::SetUp() { - _config = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true, "statereportertest")); - - _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), _config->getConfigId()); + _config = StorageConfigSet::make_storage_node_config(); + _node = std::make_unique<TestServiceLayerApp>(NodeIndex(0), _config->config_uri()); _node->setupDummyPersistence(); _clock = &_node->getClock(); _clock->setAbsoluteTimeInSeconds(1000000); @@ -90,7 +90,7 @@ void StateReporterTest::SetUp() { _filestorMetrics->initDiskMetrics(1, 1); _topSet->registerMetric(*_filestorMetrics); - _metricManager->init(config::ConfigUri(_config->getConfigId())); + _metricManager->init(_config->config_uri()); } void StateReporterTest::TearDown() { diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index 09457038b70..29f065c0157 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -1,5 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> @@ -9,9 +13,6 @@ #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/visiting/visitormanager.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> -#include <tests/common/teststorageapp.h> -#include <tests/common/testhelper.h> -#include <tests/common/dummystoragelink.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/document/test/make_bucket_space.h> #include <tests/storageserver/testvisitormessagesession.h> @@ -45,7 +46,9 @@ api::StorageMessageAddress _address(&_storage, lib::NodeType::STORAGE, 0); struct VisitorManagerTest : Test { protected: static uint32_t docCount; - std::vector<document::Document::SP > _documents; + + std::unique_ptr<StorageConfigSet> _config; + std::vector<document::Document::SP> _documents; std::unique_ptr<TestVisitorMessageSessionFactory> _messageSessionFactory; std::unique_ptr<TestServiceLayerApp> _node; std::unique_ptr<DummyStorageLink> _top; @@ -82,16 +85,16 @@ uint32_t VisitorManagerTest::docCount = 10; void VisitorManagerTest::initializeTest(bool defer_manager_thread_start) { - vdstestlib::DirConfig config(getStandardConfig(true)); - config.getConfig("stor-visitor").set("visitorthreads", "1"); + _config = StorageConfigSet::make_storage_node_config(); + _config->visitor_config().visitorthreads = 1; _messageSessionFactory = std::make_unique<TestVisitorMessageSessionFactory>(); - _node = std::make_unique<TestServiceLayerApp>(config.getConfigId()); + _node = std::make_unique<TestServiceLayerApp>(_config->config_uri()); _node->setupDummyPersistence(); _node->getStateUpdater().setClusterState(std::make_shared<lib::ClusterState>("storage:1 distributor:1")); _top = std::make_unique<DummyStorageLink>(); using vespa::config::content::core::StorVisitorConfig; - auto bootstrap_cfg = config_from<StorVisitorConfig>(config::ConfigUri(config.getConfigId())); + auto bootstrap_cfg = config_from<StorVisitorConfig>(_config->config_uri()); auto vm = std::make_unique<VisitorManager>(*bootstrap_cfg, _node->getComponentRegister(), *_messageSessionFactory, @@ -100,7 +103,7 @@ VisitorManagerTest::initializeTest(bool defer_manager_thread_start) _manager = vm.get(); _top->push_back(std::move(vm)); using StorFilestorConfig = vespa::config::content::internal::InternalStorFilestorType; - auto filestor_cfg = config_from<StorFilestorConfig>(config::ConfigUri(config.getConfigId())); + auto filestor_cfg = config_from<StorFilestorConfig>(_config->config_uri()); _top->push_back(std::make_unique<FileStorManager>(*filestor_cfg, _node->getPersistenceProvider(), _node->getComponentRegister(), *_node, _node->get_host_info())); _manager->setTimeBetweenTicks(10); diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 29a6c1baeb4..075ebd13741 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -1,5 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/dummystoragelink.h> +#include <tests/common/storage_config_set.h> +#include <tests/common/testhelper.h> +#include <tests/common/teststorageapp.h> #include <vespa/config/common/exceptions.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/document/fieldvalue/intfieldvalue.h> @@ -14,9 +18,6 @@ #include <vespa/storage/visiting/visitormanager.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/persistence.h> -#include <tests/common/testhelper.h> -#include <tests/common/teststorageapp.h> -#include <tests/common/dummystoragelink.h> #include <tests/storageserver/testvisitormessagesession.h> #include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/gtest/gtest.h> @@ -59,6 +60,8 @@ struct TestParams { struct VisitorTest : Test { static uint32_t docCount; + + std::unique_ptr<StorageConfigSet> _config; std::vector<Document::SP> _documents; std::unique_ptr<TestVisitorMessageSessionFactory> _messageSessionFactory; std::unique_ptr<TestServiceLayerApp> _node; @@ -146,24 +149,20 @@ VisitorTest::~VisitorTest() = default; void VisitorTest::initializeTest(const TestParams& params) { - vdstestlib::DirConfig config(getStandardConfig(true, "visitortest")); - config.getConfig("stor-visitor").set("visitorthreads", "1"); - config.getConfig("stor-visitor").set( - "defaultparalleliterators", - std::to_string(params._parallelBuckets)); - config.getConfig("stor-visitor").set( - "visitor_memory_usage_limit", - std::to_string(params._maxVisitorMemoryUsage)); + _config = StorageConfigSet::make_storage_node_config(); + _config->visitor_config().visitorthreads = 1; + _config->visitor_config().defaultparalleliterators = params._parallelBuckets; + _config->visitor_config().visitorMemoryUsageLimit = params._maxVisitorMemoryUsage; _messageSessionFactory = std::make_unique<TestVisitorMessageSessionFactory>(); if (params._autoReplyError.getCode() != mbus::ErrorCode::NONE) { _messageSessionFactory->_autoReplyError = params._autoReplyError; _messageSessionFactory->_createAutoReplyVisitorSessions = true; } - _node = std::make_unique<TestServiceLayerApp>(config.getConfigId()); + _node = std::make_unique<TestServiceLayerApp>(_config->config_uri()); _top = std::make_unique<DummyStorageLink>(); using vespa::config::content::core::StorVisitorConfig; - auto bootstrap_cfg = config_from<StorVisitorConfig>(config::ConfigUri(config.getConfigId())); + auto bootstrap_cfg = config_from<StorVisitorConfig>(_config->config_uri()); _top->push_back(std::unique_ptr<StorageLink>(_manager = new VisitorManager(*bootstrap_cfg, _node->getComponentRegister(), *_messageSessionFactory))); _bottom = new DummyStorageLink(); diff --git a/storageserver/src/tests/CMakeLists.txt b/storageserver/src/tests/CMakeLists.txt index 05ae89dcf89..db272ccc795 100644 --- a/storageserver/src/tests/CMakeLists.txt +++ b/storageserver/src/tests/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_executable(storageserver_gtest_runner_app TEST gtest_runner.cpp DEPENDS storageserver_storageapp + storage_testcommon vdstestlib GTest::GTest ) diff --git a/storageserver/src/tests/storageservertest.cpp b/storageserver/src/tests/storageservertest.cpp index b18b241acaa..3e5edc23464 100644 --- a/storageserver/src/tests/storageservertest.cpp +++ b/storageserver/src/tests/storageservertest.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/testhelper.h> +#include <tests/common/storage_config_set.h> #include <vespa/storage/storageserver/distributornode.h> #include <vespa/storage/storageserver/servicelayernode.h> #include <vespa/storageserver/app/distributorprocess.h> @@ -18,8 +19,8 @@ namespace storage { struct StorageServerTest : public ::testing::Test { std::unique_ptr<mbus::Slobrok> slobrok; - std::unique_ptr<vdstestlib::DirConfig> distConfig; - std::unique_ptr<vdstestlib::DirConfig> storConfig; + std::unique_ptr<StorageConfigSet> dist_config; + std::unique_ptr<StorageConfigSet> stor_config; StorageServerTest(); ~StorageServerTest() override; @@ -40,29 +41,29 @@ struct Node { virtual StorageNodeContext& getContext() = 0; }; -struct Distributor : public Node { +struct Distributor final : public Node { DistributorProcess _process; - explicit Distributor(vdstestlib::DirConfig& config); + explicit Distributor(const config::ConfigUri& config_uri); ~Distributor() override; StorageNode& getNode() override { return _process.getNode(); } StorageNodeContext& getContext() override { return _process.getContext(); } }; -struct Storage : public Node { +struct Storage final : public Node { DummyServiceLayerProcess _process; StorageComponent::UP _component; - explicit Storage(vdstestlib::DirConfig& config); + explicit Storage(const config::ConfigUri& config_uri); ~Storage() override; StorageNode& getNode() override { return _process.getNode(); } StorageNodeContext& getContext() override { return _process.getContext(); } }; -Distributor::Distributor(vdstestlib::DirConfig& config) - : _process(config::ConfigUri(config.getConfigId())) +Distributor::Distributor(const config::ConfigUri& config_uri) + : _process(config_uri) { _process.setupConfig(60000ms); _process.createNode(); @@ -70,8 +71,8 @@ Distributor::Distributor(vdstestlib::DirConfig& config) Distributor::~Distributor() = default; -Storage::Storage(vdstestlib::DirConfig& config) - : _process(config::ConfigUri(config.getConfigId())) +Storage::Storage(const config::ConfigUri& config_uri) + : _process(config_uri) { _process.setupConfig(60000ms); _process.createNode(); @@ -85,34 +86,29 @@ Storage::~Storage() = default; void StorageServerTest::SetUp() { - [[maybe_unused]] int systemResult = system("chmod -R 755 vdsroot"); - systemResult = system("rm -rf vdsroot*"); slobrok = std::make_unique<mbus::Slobrok>(); - distConfig = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(false)); - storConfig = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true)); - addSlobrokConfig(*distConfig, *slobrok); - addSlobrokConfig(*storConfig, *slobrok); - systemResult = system("mkdir -p vdsroot/disks/d0"); - systemResult = system("mkdir -p vdsroot.distributor"); + dist_config = StorageConfigSet::make_distributor_node_config(); + stor_config = StorageConfigSet::make_storage_node_config(); + dist_config->set_slobrok_config_port(slobrok->port()); + stor_config->set_slobrok_config_port(slobrok->port()); } void StorageServerTest::TearDown() { - // TODO wipe temp dirs - storConfig.reset(nullptr); - distConfig.reset(nullptr); - slobrok.reset(nullptr); + stor_config.reset(); + dist_config.reset(); + slobrok.reset(); } TEST_F(StorageServerTest, distributor_server_can_be_instantiated) { - Distributor distServer(*distConfig); + Distributor distServer(dist_config->config_uri()); } TEST_F(StorageServerTest, storage_server_can_be_instantiated) { - Storage storServer(*storConfig); + Storage storServer(stor_config->config_uri()); } } diff --git a/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.h b/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.h index f4e3d386767..38eb1404c15 100644 --- a/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.h +++ b/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.h @@ -15,12 +15,14 @@ class DummyServiceLayerProcess : public ServiceLayerProcess { std::unique_ptr<spi::PersistenceProvider> _provider; public: - DummyServiceLayerProcess(const config::ConfigUri & configUri); - ~DummyServiceLayerProcess() { shutdown(); } + explicit DummyServiceLayerProcess(const config::ConfigUri & configUri); + ~DummyServiceLayerProcess() override { + DummyServiceLayerProcess::shutdown(); + } - virtual void shutdown() override; - virtual void setupProvider() override; - virtual spi::PersistenceProvider& getProvider() override { return *_provider; } + void shutdown() override; + void setupProvider() override; + spi::PersistenceProvider& getProvider() override { return *_provider; } }; } // storage |