From 684ae2d6e3370da6d9ed1c56e29d00590c0b45de Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 3 Jun 2021 14:01:32 +0000 Subject: actually wait for connectivity OK --- configd/src/apps/sentinel/env.cpp | 116 ++++++++++++++++----------------- configd/src/apps/sentinel/env.h | 5 +- configd/src/apps/sentinel/sentinel.cpp | 20 +++--- 3 files changed, 74 insertions(+), 67 deletions(-) (limited to 'configd') diff --git a/configd/src/apps/sentinel/env.cpp b/configd/src/apps/sentinel/env.cpp index e4174ee450d..189be7a53b7 100644 --- a/configd/src/apps/sentinel/env.cpp +++ b/configd/src/apps/sentinel/env.cpp @@ -2,11 +2,12 @@ #include "env.h" #include "check-completion-handler.h" -#include "outward-check.h" +#include "connectivity.h" #include #include #include #include +#include #include #include #include @@ -18,9 +19,21 @@ using namespace std::chrono_literals; namespace config::sentinel { +namespace { + +void maybeStopNow() { + if (vespalib::SignalHandler::INT.check() || + vespalib::SignalHandler::TERM.check()) + { + throw vespalib::FatalException("got signal during boot()"); + } +} + constexpr std::chrono::milliseconds CONFIG_TIMEOUT_MS = 3min; constexpr std::chrono::milliseconds MODEL_TIMEOUT_MS = 1500ms; +} // namespace + Env::Env() : _cfgOwner(), _rpcCommandQueue(), @@ -31,6 +44,7 @@ Env::Env() _statePort(0) { _startMetrics.startedTime = vespalib::steady_clock::now(); + _stateApi.myHealth.setFailed("initializing..."); } Env::~Env() = default; @@ -38,17 +52,23 @@ Env::~Env() = default; void Env::boot(const std::string &configId) { LOG(debug, "Reading configuration for ID: %s", configId.c_str()); _cfgOwner.subscribe(configId, CONFIG_TIMEOUT_MS); - bool ok = _cfgOwner.checkForConfigUpdate(); // subscribe() should throw if something is not OK - LOG_ASSERT(ok && _cfgOwner.hasConfig()); - const auto & cfg = _cfgOwner.getConfig(); - LOG(config, "Booting sentinel '%s' with [stateserver port %d] and [rpc port %d]", - configId.c_str(), cfg.port.telnet, cfg.port.rpc); - rpcPort(cfg.port.rpc); - statePort(cfg.port.telnet); - if (auto up = ConfigOwner::fetchModelConfig(MODEL_TIMEOUT_MS)) { - waitForConnectivity(*up); + for (int retry = 0; retry < maxRetryLoops; ++retry) { + _cfgOwner.checkForConfigUpdate(); + LOG_ASSERT(_cfgOwner.hasConfig()); + const auto & cfg = _cfgOwner.getConfig(); + LOG(config, "Booting sentinel '%s' with [stateserver port %d] and [rpc port %d]", + configId.c_str(), cfg.port.telnet, cfg.port.rpc); + rpcPort(cfg.port.rpc); + statePort(cfg.port.telnet); + if (waitForConnectivity(retry)) { + _stateApi.myHealth.setOk(); + return; + } else { + LOG(warning, "Bad network connectivity, retry from start"); + } } + throw InvalidConfigException("Giving up - too many connectivity check failures"); } void Env::rpcPort(int port) { @@ -93,60 +113,40 @@ void Env::respondAsEmpty() { } } -namespace { - -const char *toString(CcResult value) { - switch (value) { - case CcResult::UNKNOWN: return "unknown"; - case CcResult::CONN_FAIL: return "failed to connect"; - case CcResult::REVERSE_FAIL: return "connect OK, but reverse check FAILED"; - case CcResult::REVERSE_UNAVAIL: return "connect OK, but reverse check unavailable"; - case CcResult::ALL_OK: return "both ways connectivity OK"; +bool Env::waitForConnectivity(int outerRetry) { + Connectivity::CheckResult lastCheckResult; + auto up = ConfigOwner::fetchModelConfig(MODEL_TIMEOUT_MS); + if (! up) { + LOG(warning, "could not get model config, skipping connectivity checks"); + return true; } - LOG(error, "Unknown CcResult enum value: %d", (int)value); - LOG_ABORT("Unknown CcResult enum value"); -} - -std::map specsFrom(const ModelConfig &model) { - std::map checkSpecs; - for (const auto & h : model.hosts) { - bool foundSentinelPort = false; - for (const auto & s : h.services) { - if (s.name == "config-sentinel") { - for (const auto & p : s.ports) { - if (p.tags.find("rpc") != p.tags.npos) { - auto spec = fmt("tcp/%s:%d", h.name.c_str(), p.number); - checkSpecs[h.name] = spec; - foundSentinelPort = true; - } + Connectivity checker(_cfgOwner.getConfig().connectivity, *_rpcServer); + for (int retry = 0; retry < maxRetriesInsideLoop; ++retry) { + auto res = checker.checkConnectivity(*up); + if (res.enoughOk) { + LOG(info, "Connectivity check OK, proceeding with service startup"); + if (! res.allOk) { + for (const std::string &detail : res.details) { + LOG(info, "Connectivity check details: %s", detail.c_str()); } } + return true; } - if (! foundSentinelPort) { - LOG(warning, "Did not find 'config-sentinel' RPC port in model for host %s [%zd services]", - h.name.c_str(), h.services.size()); + LOG(warning, "Connectivity check FAILED (try %d)", 1 + retry + maxRetriesInsideLoop*outerRetry); + _stateApi.myHealth.setFailed("FAILED connectivity check"); + if (lastCheckResult.details != res.details) { + for (const std::string &detail : res.details) { + LOG(info, "Connectivity check details: %s", detail.c_str()); + } + lastCheckResult = std::move(res); + } + for (int i = 0; i <= outerRetry; ++i) { + respondAsEmpty(); + maybeStopNow(); + std::this_thread::sleep_for(1s); } } - return checkSpecs; -} - -} - -void Env::waitForConnectivity(const ModelConfig &model) { - auto checkSpecs = specsFrom(model); - OutwardCheckContext checkContext(checkSpecs.size(), - vespa::Defaults::vespaHostname(), - _rpcServer->getPort(), - _rpcServer->orb()); - std::map connectivityMap; - for (const auto & [ hn, spec ] : checkSpecs) { - connectivityMap.try_emplace(hn, spec, checkContext); - } - checkContext.latch.await(); - for (const auto & [hostname, check] : connectivityMap) { - LOG(info, "outward check status for host %s is: %s", - hostname.c_str(), toString(check.result())); - } + return false; } } diff --git a/configd/src/apps/sentinel/env.h b/configd/src/apps/sentinel/env.h index f117854f006..9a347e8cd85 100644 --- a/configd/src/apps/sentinel/env.h +++ b/configd/src/apps/sentinel/env.h @@ -25,6 +25,9 @@ public: CommandQueue &commandQueue() { return _rpcCommandQueue; } StartMetrics &metrics() { return _startMetrics; } + static constexpr int maxRetryLoops = 5; + static constexpr int maxRetriesInsideLoop = 10; + void boot(const std::string &configId); void rpcPort(int portnum); void statePort(int portnum); @@ -32,7 +35,7 @@ public: void notifyConfigUpdated(); private: void respondAsEmpty(); - void waitForConnectivity(const ModelConfig &model); + bool waitForConnectivity(int outerRetry); ConfigOwner _cfgOwner; CommandQueue _rpcCommandQueue; std::unique_ptr _rpcServer; diff --git a/configd/src/apps/sentinel/sentinel.cpp b/configd/src/apps/sentinel/sentinel.cpp index 18d4dc28f8a..4a56ef8fd33 100644 --- a/configd/src/apps/sentinel/sentinel.cpp +++ b/configd/src/apps/sentinel/sentinel.cpp @@ -65,16 +65,20 @@ main(int argc, char **argv) LOG(debug, "Reading configuration"); try { environment.boot(configId); + } catch (vespalib::FatalException& ex) { + LOG(error, "Stopping before boot complete: %s", ex.message()); + EV_STOPPING("config-sentinel", ex.message()); + return EXIT_FAILURE; } catch (ConfigTimeoutException & ex) { - LOG(warning, "Timeout getting config, please check your setup. Will exit and restart: %s", ex.getMessage().c_str()); - EV_STOPPING("config-sentinel", ex.what()); + LOG(warning, "Timeout getting config, please check your setup. Will exit and restart: %s", ex.message()); + EV_STOPPING("config-sentinel", ex.message()); return EXIT_FAILURE; } catch (InvalidConfigException& ex) { - LOG(error, "Fatal: Invalid configuration, please check your setup: %s", ex.getMessage().c_str()); - EV_STOPPING("config-sentinel", ex.what()); + LOG(error, "Fatal: Invalid configuration, please check your setup: %s", ex.message()); + EV_STOPPING("config-sentinel", ex.message()); return EXIT_FAILURE; } catch (ConfigRuntimeException& ex) { - LOG(error, "Fatal: Could not get config, please check your setup: %s", ex.getMessage().c_str()); + LOG(error, "Fatal: Could not get config, please check your setup: %s", ex.message()); EV_STOPPING("config-sentinel", ex.what()); return EXIT_FAILURE; } @@ -86,13 +90,13 @@ main(int argc, char **argv) vespalib::SignalHandler::CHLD.clear(); manager.doWork(); // Check for child procs & commands } catch (InvalidConfigException& ex) { - LOG(warning, "Configuration problem: (ignoring): %s", ex.what()); + LOG(warning, "Configuration problem: (ignoring): %s", ex.message()); } catch (vespalib::PortListenException& ex) { - LOG(error, "Fatal: %s", ex.getMessage().c_str()); + LOG(error, "Fatal: %s", ex.message()); EV_STOPPING("config-sentinel", ex.what()); return EXIT_FAILURE; } catch (vespalib::FatalException& ex) { - LOG(error, "Fatal: %s", ex.getMessage().c_str()); + LOG(error, "Fatal: %s", ex.message()); EV_STOPPING("config-sentinel", ex.what()); return EXIT_FAILURE; } -- cgit v1.2.3 From fb725932a4a9e0b206d79282606e046761caea70 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 4 Jun 2021 13:53:16 +0000 Subject: minor fixes for detail logging --- configd/src/apps/sentinel/connectivity.cpp | 6 ++++++ configd/src/apps/sentinel/connectivity.h | 1 + configd/src/apps/sentinel/env.cpp | 12 ++++-------- 3 files changed, 11 insertions(+), 8 deletions(-) (limited to 'configd') diff --git a/configd/src/apps/sentinel/connectivity.cpp b/configd/src/apps/sentinel/connectivity.cpp index 9cced1d3475..7d9134ca51f 100644 --- a/configd/src/apps/sentinel/connectivity.cpp +++ b/configd/src/apps/sentinel/connectivity.cpp @@ -16,6 +16,12 @@ using namespace std::chrono_literals; namespace config::sentinel { +void Connectivity::CheckResult::logDetails() const { + for (const std::string &detail : details) { + LOG(info, "Connectivity check details: %s", detail.c_str()); + } +} + Connectivity::Connectivity(const SentinelConfig::Connectivity & config, RpcServer &rpcServer) : _config(config), _rpcServer(rpcServer) diff --git a/configd/src/apps/sentinel/connectivity.h b/configd/src/apps/sentinel/connectivity.h index 0e32b5243e0..69cea835da6 100644 --- a/configd/src/apps/sentinel/connectivity.h +++ b/configd/src/apps/sentinel/connectivity.h @@ -25,6 +25,7 @@ public: bool enoughOk; bool allOk; std::vector details; + void logDetails() const; }; CheckResult checkConnectivity(const ModelConfig &model); diff --git a/configd/src/apps/sentinel/env.cpp b/configd/src/apps/sentinel/env.cpp index 189be7a53b7..ded615a3c4a 100644 --- a/configd/src/apps/sentinel/env.cpp +++ b/configd/src/apps/sentinel/env.cpp @@ -114,30 +114,26 @@ void Env::respondAsEmpty() { } bool Env::waitForConnectivity(int outerRetry) { - Connectivity::CheckResult lastCheckResult; auto up = ConfigOwner::fetchModelConfig(MODEL_TIMEOUT_MS); if (! up) { LOG(warning, "could not get model config, skipping connectivity checks"); return true; } + Connectivity::CheckResult lastCheckResult; Connectivity checker(_cfgOwner.getConfig().connectivity, *_rpcServer); for (int retry = 0; retry < maxRetriesInsideLoop; ++retry) { auto res = checker.checkConnectivity(*up); if (res.enoughOk) { LOG(info, "Connectivity check OK, proceeding with service startup"); - if (! res.allOk) { - for (const std::string &detail : res.details) { - LOG(info, "Connectivity check details: %s", detail.c_str()); - } + if (retry > 0 || ! res.allOk) { + res.logDetails(); } return true; } LOG(warning, "Connectivity check FAILED (try %d)", 1 + retry + maxRetriesInsideLoop*outerRetry); _stateApi.myHealth.setFailed("FAILED connectivity check"); if (lastCheckResult.details != res.details) { - for (const std::string &detail : res.details) { - LOG(info, "Connectivity check details: %s", detail.c_str()); - } + res.logDetails(); lastCheckResult = std::move(res); } for (int i = 0; i <= outerRetry; ++i) { -- cgit v1.2.3 From f16f846f7486febb25069284278034ce597dc164 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 7 Jun 2021 10:26:32 +0000 Subject: simplify after review --- configd/src/apps/sentinel/connectivity.cpp | 62 ++++++++++++++------------- configd/src/apps/sentinel/connectivity.h | 20 +++------ configd/src/apps/sentinel/env.cpp | 68 +++++++++++------------------- configd/src/apps/sentinel/env.h | 4 -- 4 files changed, 64 insertions(+), 90 deletions(-) (limited to 'configd') diff --git a/configd/src/apps/sentinel/connectivity.cpp b/configd/src/apps/sentinel/connectivity.cpp index 7d9134ca51f..74f11f086d6 100644 --- a/configd/src/apps/sentinel/connectivity.cpp +++ b/configd/src/apps/sentinel/connectivity.cpp @@ -1,5 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "config-owner.h" #include "connectivity.h" #include "outward-check.h" #include @@ -16,20 +17,9 @@ using namespace std::chrono_literals; namespace config::sentinel { -void Connectivity::CheckResult::logDetails() const { - for (const std::string &detail : details) { - LOG(info, "Connectivity check details: %s", detail.c_str()); - } -} - -Connectivity::Connectivity(const SentinelConfig::Connectivity & config, RpcServer &rpcServer) - : _config(config), - _rpcServer(rpcServer) -{ - LOG(config, "connectivity.maxBadReverseCount = %d", _config.maxBadReverseCount); - LOG(config, "connectivity.maxBadOutPercent = %d", _config.maxBadOutPercent); -} +constexpr std::chrono::milliseconds MODEL_TIMEOUT_MS = 60s; +Connectivity::Connectivity() = default; Connectivity::~Connectivity() = default; namespace { @@ -71,16 +61,28 @@ std::map specsFrom(const ModelConfig &model) { } -Connectivity::CheckResult -Connectivity::checkConnectivity(const ModelConfig &model) { - const auto checkSpecs = specsFrom(model); - size_t clusterSize = checkSpecs.size(); +void Connectivity::configure(const SentinelConfig::Connectivity &config) { + _config = config; + LOG(config, "connectivity.maxBadReverseCount = %d", _config.maxBadReverseCount); + LOG(config, "connectivity.maxBadOutPercent = %d", _config.maxBadOutPercent); + if (auto up = ConfigOwner::fetchModelConfig(MODEL_TIMEOUT_MS)) { + _checkSpecs = specsFrom(*up); + } +} + +bool +Connectivity::checkConnectivity(RpcServer &rpcServer) { + size_t clusterSize = _checkSpecs.size(); + if (clusterSize == 0) { + LOG(warning, "could not get model config, skipping connectivity checks"); + return true; + } OutwardCheckContext checkContext(clusterSize, vespa::Defaults::vespaHostname(), - _rpcServer.getPort(), - _rpcServer.orb()); + rpcServer.getPort(), + rpcServer.orb()); std::map connectivityMap; - for (const auto & [ hn, spec ] : checkSpecs) { + for (const auto & [ hn, spec ] : _checkSpecs) { connectivityMap.try_emplace(hn, spec, checkContext); } checkContext.latch.await(); @@ -88,6 +90,12 @@ Connectivity::checkConnectivity(const ModelConfig &model) { size_t numFailedReverse = 0; bool allChecksOk = true; for (const auto & [hostname, check] : connectivityMap) { + const char *detail = toString(check.result()); + std::string prev = _detailsPerHost[hostname]; + if (prev != detail) { + LOG(info, "Connectivity check details: %s -> %s", hostname.c_str(), detail); + } + _detailsPerHost[hostname] = detail; LOG_ASSERT(check.result() != CcResult::UNKNOWN); if (check.result() == CcResult::CONN_FAIL) ++numFailedConns; if (check.result() == CcResult::REVERSE_FAIL) ++numFailedReverse; @@ -103,16 +111,12 @@ Connectivity::checkConnectivity(const ModelConfig &model) { numFailedConns, clusterSize, pct, _config.maxBadOutPercent); allChecksOk = false; } - std::vector details; - for (const auto & [hostname, check] : connectivityMap) { - std::string detail = fmt("%s -> %s", hostname.c_str(), toString(check.result())); - details.push_back(detail); + if (allChecksOk && (numFailedConns == 0) && (numFailedReverse == 0)) { + LOG(info, "All connectivity checks OK, proceeding with service startup"); + } else if (allChecksOk) { + LOG(info, "Enough connectivity checks OK, proceeding with service startup"); } - CheckResult result{false, false, {}}; - result.enoughOk = allChecksOk; - result.allOk = (numFailedConns == 0) && (numFailedReverse == 0); - result.details = std::move(details); - return result; + return allChecksOk; } } diff --git a/configd/src/apps/sentinel/connectivity.h b/configd/src/apps/sentinel/connectivity.h index 69cea835da6..1c7ee8ddc57 100644 --- a/configd/src/apps/sentinel/connectivity.h +++ b/configd/src/apps/sentinel/connectivity.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include using cloud::config::SentinelConfig; using cloud::config::ModelConfig; @@ -18,20 +18,14 @@ namespace config::sentinel { **/ class Connectivity { public: - Connectivity(const SentinelConfig::Connectivity & config, RpcServer &rpcServer); + Connectivity(); ~Connectivity(); - - struct CheckResult { - bool enoughOk; - bool allOk; - std::vector details; - void logDetails() const; - }; - - CheckResult checkConnectivity(const ModelConfig &model); + void configure(const SentinelConfig::Connectivity &config); + bool checkConnectivity(RpcServer &rpcServer); private: - const SentinelConfig::Connectivity _config; - RpcServer &_rpcServer; + SentinelConfig::Connectivity _config; + std::map _checkSpecs; + std::map _detailsPerHost; }; } diff --git a/configd/src/apps/sentinel/env.cpp b/configd/src/apps/sentinel/env.cpp index ded615a3c4a..9763956bada 100644 --- a/configd/src/apps/sentinel/env.cpp +++ b/configd/src/apps/sentinel/env.cpp @@ -30,7 +30,7 @@ void maybeStopNow() { } constexpr std::chrono::milliseconds CONFIG_TIMEOUT_MS = 3min; -constexpr std::chrono::milliseconds MODEL_TIMEOUT_MS = 1500ms; +constexpr int maxConnectivityRetries = 100; } // namespace @@ -53,22 +53,34 @@ void Env::boot(const std::string &configId) { LOG(debug, "Reading configuration for ID: %s", configId.c_str()); _cfgOwner.subscribe(configId, CONFIG_TIMEOUT_MS); // subscribe() should throw if something is not OK - for (int retry = 0; retry < maxRetryLoops; ++retry) { - _cfgOwner.checkForConfigUpdate(); - LOG_ASSERT(_cfgOwner.hasConfig()); - const auto & cfg = _cfgOwner.getConfig(); - LOG(config, "Booting sentinel '%s' with [stateserver port %d] and [rpc port %d]", - configId.c_str(), cfg.port.telnet, cfg.port.rpc); - rpcPort(cfg.port.rpc); - statePort(cfg.port.telnet); - if (waitForConnectivity(retry)) { + Connectivity checker; + for (int retry = 0; retry < maxConnectivityRetries; ++retry) { + bool changed = _cfgOwner.checkForConfigUpdate(); + if (changed) { + LOG_ASSERT(_cfgOwner.hasConfig()); + const auto & cfg = _cfgOwner.getConfig(); + LOG(config, "Booting sentinel '%s' with [stateserver port %d] and [rpc port %d]", + configId.c_str(), cfg.port.telnet, cfg.port.rpc); + rpcPort(cfg.port.rpc); + statePort(cfg.port.telnet); + checker.configure(cfg.connectivity); + } + if (checker.checkConnectivity(*_rpcServer)) { _stateApi.myHealth.setOk(); return; } else { - LOG(warning, "Bad network connectivity, retry from start"); + _stateApi.myHealth.setFailed("FAILED connectivity check"); + if ((retry % 10) == 0) { + LOG(warning, "Bad network connectivity (try %d)", 1+retry); + } + for (int i = 0; i < 5; ++i) { + respondAsEmpty(); + maybeStopNow(); + std::this_thread::sleep_for(600ms); + } } } - throw InvalidConfigException("Giving up - too many connectivity check failures"); + throw vespalib::FatalException("Giving up - too many connectivity check failures"); } void Env::rpcPort(int port) { @@ -113,36 +125,4 @@ void Env::respondAsEmpty() { } } -bool Env::waitForConnectivity(int outerRetry) { - auto up = ConfigOwner::fetchModelConfig(MODEL_TIMEOUT_MS); - if (! up) { - LOG(warning, "could not get model config, skipping connectivity checks"); - return true; - } - Connectivity::CheckResult lastCheckResult; - Connectivity checker(_cfgOwner.getConfig().connectivity, *_rpcServer); - for (int retry = 0; retry < maxRetriesInsideLoop; ++retry) { - auto res = checker.checkConnectivity(*up); - if (res.enoughOk) { - LOG(info, "Connectivity check OK, proceeding with service startup"); - if (retry > 0 || ! res.allOk) { - res.logDetails(); - } - return true; - } - LOG(warning, "Connectivity check FAILED (try %d)", 1 + retry + maxRetriesInsideLoop*outerRetry); - _stateApi.myHealth.setFailed("FAILED connectivity check"); - if (lastCheckResult.details != res.details) { - res.logDetails(); - lastCheckResult = std::move(res); - } - for (int i = 0; i <= outerRetry; ++i) { - respondAsEmpty(); - maybeStopNow(); - std::this_thread::sleep_for(1s); - } - } - return false; -} - } diff --git a/configd/src/apps/sentinel/env.h b/configd/src/apps/sentinel/env.h index 9a347e8cd85..f71fb537068 100644 --- a/configd/src/apps/sentinel/env.h +++ b/configd/src/apps/sentinel/env.h @@ -25,9 +25,6 @@ public: CommandQueue &commandQueue() { return _rpcCommandQueue; } StartMetrics &metrics() { return _startMetrics; } - static constexpr int maxRetryLoops = 5; - static constexpr int maxRetriesInsideLoop = 10; - void boot(const std::string &configId); void rpcPort(int portnum); void statePort(int portnum); @@ -35,7 +32,6 @@ public: void notifyConfigUpdated(); private: void respondAsEmpty(); - bool waitForConnectivity(int outerRetry); ConfigOwner _cfgOwner; CommandQueue _rpcCommandQueue; std::unique_ptr _rpcServer; -- cgit v1.2.3 From a402fb4fab902b9f8c9a8859b1142e436bf439e3 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 7 Jun 2021 21:07:23 +0000 Subject: minor fixups after review --- configd/src/apps/sentinel/connectivity.cpp | 6 +++--- configd/src/apps/sentinel/env.cpp | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'configd') diff --git a/configd/src/apps/sentinel/connectivity.cpp b/configd/src/apps/sentinel/connectivity.cpp index 74f11f086d6..132b57fc884 100644 --- a/configd/src/apps/sentinel/connectivity.cpp +++ b/configd/src/apps/sentinel/connectivity.cpp @@ -24,7 +24,7 @@ Connectivity::~Connectivity() = default; namespace { -const char *toString(CcResult value) { +std::string toString(CcResult value) { switch (value) { case CcResult::UNKNOWN: return "BAD: missing result"; // very very bad case CcResult::REVERSE_FAIL: return "connect OK, but reverse check FAILED"; // very bad @@ -90,10 +90,10 @@ Connectivity::checkConnectivity(RpcServer &rpcServer) { size_t numFailedReverse = 0; bool allChecksOk = true; for (const auto & [hostname, check] : connectivityMap) { - const char *detail = toString(check.result()); + std::string detail = toString(check.result()); std::string prev = _detailsPerHost[hostname]; if (prev != detail) { - LOG(info, "Connectivity check details: %s -> %s", hostname.c_str(), detail); + LOG(info, "Connectivity check details: %s -> %s", hostname.c_str(), detail.c_str()); } _detailsPerHost[hostname] = detail; LOG_ASSERT(check.result() != CcResult::UNKNOWN); diff --git a/configd/src/apps/sentinel/env.cpp b/configd/src/apps/sentinel/env.cpp index 9763956bada..5bbbfd8f0bd 100644 --- a/configd/src/apps/sentinel/env.cpp +++ b/configd/src/apps/sentinel/env.cpp @@ -56,6 +56,7 @@ void Env::boot(const std::string &configId) { Connectivity checker; for (int retry = 0; retry < maxConnectivityRetries; ++retry) { bool changed = _cfgOwner.checkForConfigUpdate(); + LOG_ASSERT(changed || retry > 0); if (changed) { LOG_ASSERT(_cfgOwner.hasConfig()); const auto & cfg = _cfgOwner.getConfig(); -- cgit v1.2.3