Propagate existing StorageNode config from main Process reconfig loop

6 files changed, 106 insertions, 24 deletions

diff --git a/storageserver/src/tests/storageservertest.cpp b/storageserver/src/tests/storageservertest.cpp
index 2633449f28a..b18b241acaa 100644
--- a/storageserver/src/tests/storageservertest.cpp
+++ b/storageserver/src/tests/storageservertest.cpp
@@ -43,7 +43,7 @@ struct Node {
 struct Distributor : public Node {
     DistributorProcess _process;
 
-    Distributor(vdstestlib::DirConfig& config);
+    explicit Distributor(vdstestlib::DirConfig& config);
     ~Distributor() override;
 
     StorageNode& getNode() override { return _process.getNode(); }
@@ -54,7 +54,7 @@ struct Storage : public Node {
     DummyServiceLayerProcess _process;
     StorageComponent::UP _component;
 
-    Storage(vdstestlib::DirConfig& config);
+    explicit Storage(vdstestlib::DirConfig& config);
     ~Storage() override;
 
     StorageNode& getNode() override { return _process.getNode(); }
@@ -75,8 +75,7 @@ Storage::Storage(vdstestlib::DirConfig& config)
 {
     _process.setupConfig(60000ms);
     _process.createNode();
-    _component = std::make_unique<StorageComponent>(
-    getContext().getComponentRegister(), "test");
+    _component = std::make_unique<StorageComponent>(getContext().getComponentRegister(), "test");
 }
 
 Storage::~Storage() = default;
@@ -93,7 +92,6 @@ StorageServerTest::SetUp()
     storConfig = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true));
     addSlobrokConfig(*distConfig, *slobrok);
     addSlobrokConfig(*storConfig, *slobrok);
-    storConfig->getConfig("stor-filestor").set("fail_disk_after_error_count", "1");
     systemResult = system("mkdir -p vdsroot/disks/d0");
     systemResult = system("mkdir -p vdsroot.distributor");
 }
@@ -101,6 +99,7 @@ StorageServerTest::SetUp()
 void
 StorageServerTest::TearDown()
 {
+    // TODO wipe temp dirs
     storConfig.reset(nullptr);
     distConfig.reset(nullptr);
     slobrok.reset(nullptr);
diff --git a/storageserver/src/vespa/storageserver/app/distributorprocess.cpp b/storageserver/src/vespa/storageserver/app/distributorprocess.cpp
index aad813a49fc..b56a4e1884b 100644
--- a/storageserver/src/vespa/storageserver/app/distributorprocess.cpp
+++ b/storageserver/src/vespa/storageserver/app/distributorprocess.cpp
@@ -61,9 +61,6 @@ DistributorProcess::setupConfig(vespalib::duration subscribeTimeout)
     using vespa::config::content::core::StorDistributormanagerConfig;
     using vespa::config::content::core::StorVisitordispatcherConfig;
 
-    auto distr_cfg = config::ConfigGetter<StorDistributormanagerConfig>::getConfig(
-            _configUri.getConfigId(), _configUri.getContext(), subscribeTimeout);
-    _num_distributor_stripes = adjusted_num_distributor_stripes(distr_cfg->numDistributorStripes);
     _distributorConfigHandler = _configSubscriber.subscribe<StorDistributormanagerConfig>(_configUri.getConfigId(), subscribeTimeout);
     _visitDispatcherConfigHandler = _configSubscriber.subscribe<StorVisitordispatcherConfig>(_configUri.getConfigId(), subscribeTimeout);
     Process::setupConfig(subscribeTimeout);
@@ -99,8 +96,19 @@ DistributorProcess::configUpdated()
 void
 DistributorProcess::createNode()
 {
-    _node = std::make_unique<DistributorNode>(_configUri, _context, *this, _num_distributor_stripes, StorageLink::UP(), std::move(_storage_chain_builder));
-    _node->handleConfigChange(*_distributorConfigHandler->getConfig());
+    auto distributor_config = _distributorConfigHandler->getConfig();
+    _num_distributor_stripes = adjusted_num_distributor_stripes(distributor_config->numDistributorStripes);
+    // TODO dedupe, consolidate
+    StorageNode::BootstrapConfigs bc;
+    bc.bucket_spaces_cfg = _bucket_spaces_cfg_handle->getConfig();
+    bc.bouncer_cfg = _bouncer_cfg_handle->getConfig();
+    bc.comm_mgr_cfg = _comm_mgr_cfg_handle->getConfig();
+    bc.distribution_cfg = _distribution_cfg_handle->getConfig();
+    bc.server_cfg = _server_cfg_handle->getConfig();
+
+    _node = std::make_unique<DistributorNode>(_configUri, _context, std::move(bc), *this, _num_distributor_stripes,
+                                              StorageLink::UP(), std::move(_storage_chain_builder));
+    _node->handleConfigChange(*distributor_config);
     _node->handleConfigChange(*_visitDispatcherConfigHandler->getConfig());
 }
 
diff --git a/storageserver/src/vespa/storageserver/app/process.cpp b/storageserver/src/vespa/storageserver/app/process.cpp
index 4b1586032e4..87b20f9ec2e 100644
--- a/storageserver/src/vespa/storageserver/app/process.cpp
+++ b/storageserver/src/vespa/storageserver/app/process.cpp
@@ -1,11 +1,12 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "process.h"
+
+#include <vespa/config/subscription/configsubscriber.hpp>
 #include <vespa/document/repo/document_type_repo_factory.h>
 #include <vespa/storage/storageserver/storagenode.h>
 #include <vespa/storage/storageserver/storagenodecontext.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include <vespa/config/subscription/configsubscriber.hpp>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".process");
@@ -24,11 +25,17 @@ Process::~Process() = default;
 void
 Process::setupConfig(vespalib::duration subscribeTimeout)
 {
-    _documentHandler = _configSubscriber.subscribe<document::config::DocumenttypesConfig>(_configUri.getConfigId(), subscribeTimeout);
+    _document_cfg_handle      = _configSubscriber.subscribe<DocumentTypesConfig>(_configUri.getConfigId(), subscribeTimeout);
+    _bucket_spaces_cfg_handle = _configSubscriber.subscribe<BucketspacesConfig>(_configUri.getConfigId(), subscribeTimeout);
+    _comm_mgr_cfg_handle      = _configSubscriber.subscribe<CommunicationManagerConfig>(_configUri.getConfigId(), subscribeTimeout);
+    _bouncer_cfg_handle       = _configSubscriber.subscribe<StorBouncerConfig>(_configUri.getConfigId(), subscribeTimeout);
+    _distribution_cfg_handle  = _configSubscriber.subscribe<StorDistributionConfig>(_configUri.getConfigId(), subscribeTimeout);
+    _server_cfg_handle        = _configSubscriber.subscribe<StorServerConfig>(_configUri.getConfigId(), subscribeTimeout);
+
     if (!_configSubscriber.nextConfig()) {
-        throw vespalib::TimeoutException("Could not subscribe to document config within timeout");
+        throw vespalib::TimeoutException("Could not subscribe to configs within timeout");
     }
-    _repos.push_back(DocumentTypeRepoFactory::make(*_documentHandler->getConfig()));
+    _repos.push_back(DocumentTypeRepoFactory::make(*_document_cfg_handle->getConfig()));
     getContext().getComponentRegister().setDocumentTypeRepo(_repos.back());
 }
 
@@ -36,26 +43,46 @@ bool
 Process::configUpdated()
 {
     _configSubscriber.nextGenerationNow();
-    if (_documentHandler->isChanged()) {
+    if (_document_cfg_handle->isChanged()) {
         LOG(info, "Document config detected changed");
         return true;
     }
-    return false;
+    bool changed = (_bucket_spaces_cfg_handle->isChanged()
+                 || _comm_mgr_cfg_handle->isChanged()
+                 || _bouncer_cfg_handle->isChanged()
+                 || _distribution_cfg_handle->isChanged()
+                 || _server_cfg_handle->isChanged());
+    return changed;
 }
 
 void
 Process::updateConfig()
 {
-    if (_documentHandler->isChanged()) {
-        _repos.push_back(DocumentTypeRepoFactory::make(*_documentHandler->getConfig()));
+    if (_document_cfg_handle->isChanged()) {
+        _repos.push_back(DocumentTypeRepoFactory::make(*_document_cfg_handle->getConfig()));
         getNode().setNewDocumentRepo(_repos.back());
     }
+    if (_bucket_spaces_cfg_handle->isChanged()) {
+        getNode().configure(_bucket_spaces_cfg_handle->getConfig());
+    }
+    if (_comm_mgr_cfg_handle->isChanged()) {
+        getNode().configure(_comm_mgr_cfg_handle->getConfig());
+    }
+    if (_bouncer_cfg_handle->isChanged()) {
+        getNode().configure(_bouncer_cfg_handle->getConfig());
+    }
+    if (_distribution_cfg_handle->isChanged()) {
+        getNode().configure(_distribution_cfg_handle->getConfig());
+    }
+    if (_server_cfg_handle->isChanged()) {
+        getNode().configure(_server_cfg_handle->getConfig());
+    }
 }
 
 void
 Process::shutdown()
 {
-    removeConfigSubscriptions();
+    removeConfigSubscriptions(); // TODO remove? unused
 }
 
 int64_t
diff --git a/storageserver/src/vespa/storageserver/app/process.h b/storageserver/src/vespa/storageserver/app/process.h
index e70427c6e04..a30611b78c8 100644
--- a/storageserver/src/vespa/storageserver/app/process.h
+++ b/storageserver/src/vespa/storageserver/app/process.h
@@ -14,10 +14,15 @@
 
 #pragma once
 
+#include <vespa/config-bucketspaces.h>
+#include <vespa/config-stor-distribution.h>
+#include <vespa/config/subscription/configsubscriber.h>
+#include <vespa/config/subscription/configuri.h>
 #include <vespa/document/config/config-documenttypes.h>
+#include <vespa/storage/config/config-stor-bouncer.h>
+#include <vespa/storage/config/config-stor-communicationmanager.h>
+#include <vespa/storage/config/config-stor-server.h>
 #include <vespa/storage/storageserver/applicationgenerationfetcher.h>
-#include <vespa/config/subscription/configuri.h>
-#include <vespa/config/subscription/configsubscriber.h>
 
 namespace document { class DocumentTypeRepo; }
 
@@ -28,19 +33,32 @@ struct StorageNodeContext;
 
 class Process : public ApplicationGenerationFetcher {
 protected:
+    using DocumentTypesConfig        = document::config::DocumenttypesConfig;
+    using BucketspacesConfig         = vespa::config::content::core::BucketspacesConfig;
+    using CommunicationManagerConfig = vespa::config::content::core::StorCommunicationmanagerConfig;
+    using StorBouncerConfig          = vespa::config::content::core::StorBouncerConfig;
+    using StorDistributionConfig     = vespa::config::content::StorDistributionConfig;
+    using StorServerConfig           = vespa::config::content::core::StorServerConfig;
+
     using DocumentTypeRepoSP = std::shared_ptr<const document::DocumentTypeRepo>;
     config::ConfigUri _configUri;
     DocumentTypeRepoSP getTypeRepo() { return _repos.back(); }
     config::ConfigSubscriber _configSubscriber;
 
+    std::unique_ptr<config::ConfigHandle<DocumentTypesConfig>>        _document_cfg_handle;
+    std::unique_ptr<config::ConfigHandle<BucketspacesConfig>>         _bucket_spaces_cfg_handle;
+    std::unique_ptr<config::ConfigHandle<CommunicationManagerConfig>> _comm_mgr_cfg_handle;
+    std::unique_ptr<config::ConfigHandle<StorBouncerConfig>>          _bouncer_cfg_handle;
+    std::unique_ptr<config::ConfigHandle<StorDistributionConfig>>     _distribution_cfg_handle;
+    std::unique_ptr<config::ConfigHandle<StorServerConfig>>           _server_cfg_handle;
+
 private:
-    config::ConfigHandle<document::config::DocumenttypesConfig>::UP _documentHandler;
     std::vector<DocumentTypeRepoSP> _repos;
 
 public:
     using UP = std::unique_ptr<Process>;
 
-    Process(const config::ConfigUri & configUri);
+    explicit Process(const config::ConfigUri & configUri);
     ~Process() override;
 
     virtual void setupConfig(vespalib::duration subscribeTimeout);
diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp
index 369cca4b166..6b713b8e3f4 100644
--- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp
+++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp
@@ -51,11 +51,37 @@ ServiceLayerProcess::shutdown()
 }
 
 void
+ServiceLayerProcess::setupConfig(vespalib::duration subscribe_timeout)
+{
+    Process::setupConfig(subscribe_timeout);
+}
+
+void
+ServiceLayerProcess::updateConfig()
+{
+    Process::updateConfig();
+}
+
+bool
+ServiceLayerProcess::configUpdated()
+{
+    return Process::configUpdated();
+}
+
+void
 ServiceLayerProcess::createNode()
 {
     add_external_visitors();
     setupProvider();
-    _node = std::make_unique<ServiceLayerNode>(_configUri, _context, *this, getProvider(), _externalVisitors);
+    // TODO dedupe, consolidate
+    StorageNode::BootstrapConfigs bc;
+    bc.bucket_spaces_cfg = _bucket_spaces_cfg_handle->getConfig();
+    bc.bouncer_cfg = _bouncer_cfg_handle->getConfig();
+    bc.comm_mgr_cfg = _comm_mgr_cfg_handle->getConfig();
+    bc.distribution_cfg = _distribution_cfg_handle->getConfig();
+    bc.server_cfg = _server_cfg_handle->getConfig();
+
+    _node = std::make_unique<ServiceLayerNode>(_configUri, _context, std::move(bc), *this, getProvider(), _externalVisitors);
     if (_storage_chain_builder) {
         _node->set_storage_chain_builder(std::move(_storage_chain_builder));
     }
diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h
index f95b952a68d..7e981748d1a 100644
--- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h
+++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h
@@ -45,6 +45,10 @@ public:
 
     void shutdown() override;
 
+    void setupConfig(vespalib::duration subscribe_timeout) override;
+    bool configUpdated() override;
+    void updateConfig() override;
+
     virtual void setupProvider() = 0;
     virtual spi::PersistenceProvider& getProvider() = 0;