more restart penalty

* set max restart penalty to 30 minutes
* scale much faster (exponentially) to max restart penalty
* increment restart penalty when a service needs restarting
  before 30 minutes have passed
* reset restart penalty when a service was OK for 5 hours

2 files changed, 10 insertions, 7 deletions

diff --git a/configd/src/apps/sentinel/service.cpp b/configd/src/apps/sentinel/service.cpp
index e38328975dc..530f852e80a 100644
--- a/configd/src/apps/sentinel/service.cpp
+++ b/configd/src/apps/sentinel/service.cpp
@@ -167,10 +167,13 @@ Service::start()
     // make sure the service does not restart in a tight loop:
     time_t now = time(0);
     int diff = now - _last_start;
-    if (diff < 10) {
+    if (diff < MAX_RESTART_PENALTY) {
         incrementRestartPenalty();
-        now += _restartPenalty; // will delay start this much
     }
+    if (diff > 10 * MAX_RESTART_PENALTY) {
+        resetRestartPenalty();
+    }
+    now += _restartPenalty; // will delay start this much
     _last_start = now;
 
 // make a pipe, close the good ends of it, mark it close-on-exec
@@ -230,7 +233,7 @@ Service::start()
             kill(getpid(), SIGTERM);
         }
         if (_restartPenalty > 0) {
-            LOG(debug, "%s: Applying %u sec restart penalty", name().c_str(),
+            LOG(info, "%s: Applying %u sec restart penalty", name().c_str(),
                 _restartPenalty);
             sleep(_restartPenalty);
         }
@@ -424,9 +427,9 @@ Service::setAutomatic(bool autoStatus)
 void
 Service::incrementRestartPenalty()
 {
-    if (_restartPenalty < MAX_RESTART_PENALTY) {
-        _restartPenalty++;
-    } else {
+    _restartPenalty += 1;
+    _restartPenalty *= 2;
+    if (_restartPenalty > MAX_RESTART_PENALTY) {
         _restartPenalty = MAX_RESTART_PENALTY;
     }
 }
diff --git a/configd/src/apps/sentinel/service.h b/configd/src/apps/sentinel/service.h
index 6419fdd7268..54bf1105a77 100644
--- a/configd/src/apps/sentinel/service.h
+++ b/configd/src/apps/sentinel/service.h
@@ -26,7 +26,7 @@ private:
     SentinelConfig::Service *_config;
     bool _isAutomatic;
 
-    static const unsigned int MAX_RESTART_PENALTY = 60;
+    static const int MAX_RESTART_PENALTY = 1800;
     unsigned int _restartPenalty;
     time_t _last_start;