diff options
author | Arne Juul <arnej@yahoo-inc.com> | 2018-08-20 13:04:29 +0000 |
---|---|---|
committer | Arne Juul <arnej@yahoo-inc.com> | 2018-08-21 09:55:49 +0000 |
commit | bd87e1b6146a211f8e62db6402cf337723472467 (patch) | |
tree | 6e68a92b7b1aa696cc40e6612a87813fa2c03f7a | |
parent | 7aa39dd1e650c88989dec9d22188522603f2124e (diff) |
more restart penalty
* set max restart penalty to 30 minutes
* scale much faster (exponentially) to max restart penalty
* increment restart penalty when a service needs restarting
before 30 minutes have passed
* reset restart penalty when a service was OK for 5 hours
-rw-r--r-- | configd/src/apps/sentinel/service.cpp | 15 | ||||
-rw-r--r-- | configd/src/apps/sentinel/service.h | 2 |
2 files changed, 10 insertions, 7 deletions
diff --git a/configd/src/apps/sentinel/service.cpp b/configd/src/apps/sentinel/service.cpp index e38328975dc..530f852e80a 100644 --- a/configd/src/apps/sentinel/service.cpp +++ b/configd/src/apps/sentinel/service.cpp @@ -167,10 +167,13 @@ Service::start() // make sure the service does not restart in a tight loop: time_t now = time(0); int diff = now - _last_start; - if (diff < 10) { + if (diff < MAX_RESTART_PENALTY) { incrementRestartPenalty(); - now += _restartPenalty; // will delay start this much } + if (diff > 10 * MAX_RESTART_PENALTY) { + resetRestartPenalty(); + } + now += _restartPenalty; // will delay start this much _last_start = now; // make a pipe, close the good ends of it, mark it close-on-exec @@ -230,7 +233,7 @@ Service::start() kill(getpid(), SIGTERM); } if (_restartPenalty > 0) { - LOG(debug, "%s: Applying %u sec restart penalty", name().c_str(), + LOG(info, "%s: Applying %u sec restart penalty", name().c_str(), _restartPenalty); sleep(_restartPenalty); } @@ -424,9 +427,9 @@ Service::setAutomatic(bool autoStatus) void Service::incrementRestartPenalty() { - if (_restartPenalty < MAX_RESTART_PENALTY) { - _restartPenalty++; - } else { + _restartPenalty += 1; + _restartPenalty *= 2; + if (_restartPenalty > MAX_RESTART_PENALTY) { _restartPenalty = MAX_RESTART_PENALTY; } } diff --git a/configd/src/apps/sentinel/service.h b/configd/src/apps/sentinel/service.h index 6419fdd7268..54bf1105a77 100644 --- a/configd/src/apps/sentinel/service.h +++ b/configd/src/apps/sentinel/service.h @@ -26,7 +26,7 @@ private: SentinelConfig::Service *_config; bool _isAutomatic; - static const unsigned int MAX_RESTART_PENALTY = 60; + static const int MAX_RESTART_PENALTY = 1800; unsigned int _restartPenalty; time_t _last_start; |