From bd87e1b6146a211f8e62db6402cf337723472467 Mon Sep 17 00:00:00 2001
From: Arne Juul <arnej@yahoo-inc.com>
Date: Mon, 20 Aug 2018 13:04:29 +0000
Subject: more restart penalty

* set max restart penalty to 30 minutes
* scale much faster (exponentially) to max restart penalty
* increment restart penalty when a service needs restarting
  before 30 minutes have passed
* reset restart penalty when a service was OK for 5 hours
---
 configd/src/apps/sentinel/service.cpp | 15 +++++++++------
 configd/src/apps/sentinel/service.h   |  2 +-
 2 files changed, 10 insertions(+), 7 deletions(-)

(limited to 'configd')

diff --git a/configd/src/apps/sentinel/service.cpp b/configd/src/apps/sentinel/service.cpp
index e38328975dc..530f852e80a 100644
--- a/configd/src/apps/sentinel/service.cpp
+++ b/configd/src/apps/sentinel/service.cpp
@@ -167,10 +167,13 @@ Service::start()
     // make sure the service does not restart in a tight loop:
     time_t now = time(0);
     int diff = now - _last_start;
-    if (diff < 10) {
+    if (diff < MAX_RESTART_PENALTY) {
         incrementRestartPenalty();
-        now += _restartPenalty; // will delay start this much
     }
+    if (diff > 10 * MAX_RESTART_PENALTY) {
+        resetRestartPenalty();
+    }
+    now += _restartPenalty; // will delay start this much
     _last_start = now;
 
 // make a pipe, close the good ends of it, mark it close-on-exec
@@ -230,7 +233,7 @@ Service::start()
             kill(getpid(), SIGTERM);
         }
         if (_restartPenalty > 0) {
-            LOG(debug, "%s: Applying %u sec restart penalty", name().c_str(),
+            LOG(info, "%s: Applying %u sec restart penalty", name().c_str(),
                 _restartPenalty);
             sleep(_restartPenalty);
         }
@@ -424,9 +427,9 @@ Service::setAutomatic(bool autoStatus)
 void
 Service::incrementRestartPenalty()
 {
-    if (_restartPenalty < MAX_RESTART_PENALTY) {
-        _restartPenalty++;
-    } else {
+    _restartPenalty += 1;
+    _restartPenalty *= 2;
+    if (_restartPenalty > MAX_RESTART_PENALTY) {
         _restartPenalty = MAX_RESTART_PENALTY;
     }
 }
diff --git a/configd/src/apps/sentinel/service.h b/configd/src/apps/sentinel/service.h
index 6419fdd7268..54bf1105a77 100644
--- a/configd/src/apps/sentinel/service.h
+++ b/configd/src/apps/sentinel/service.h
@@ -26,7 +26,7 @@ private:
     SentinelConfig::Service *_config;
     bool _isAutomatic;
 
-    static const unsigned int MAX_RESTART_PENALTY = 60;
+    static const int MAX_RESTART_PENALTY = 1800;
     unsigned int _restartPenalty;
     time_t _last_start;
 
-- 
cgit v1.2.3