From ecfe4675fb229c03cf446ae6448ede715e335442 Mon Sep 17 00:00:00 2001
From: Henning Baldersheim <balder@yahoo-inc.com>
Date: Fri, 24 Sep 2021 07:15:54 +0200
Subject: - Shorten grace period from 30m to 5m. - Check to remove every second
 to get rid of deactivated instances sooner.

---
 .../com/yahoo/jdisc/core/ContainerWatchdog.java    | 40 +++++++++++++++-------
 1 file changed, 27 insertions(+), 13 deletions(-)

(limited to 'jdisc_core/src/main')

diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerWatchdog.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerWatchdog.java
index 30aa0028465..7e82955eede 100644
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerWatchdog.java
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerWatchdog.java
@@ -22,8 +22,9 @@ import java.util.logging.Logger;
  */
 class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable {
 
-    static final Duration GRACE_PERIOD = Duration.ofMinutes(30);
+    static final Duration GRACE_PERIOD = Duration.ofMinutes(5);
     static final Duration UPDATE_PERIOD = Duration.ofMinutes(5);
+    static final Duration CHECK_PERIOD = Duration.ofSeconds(1);
 
     private static final Logger log = Logger.getLogger(ContainerWatchdog.class.getName());
 
@@ -35,6 +36,7 @@ class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable {
     private ActiveContainer currentContainer;
     private Instant currentContainerActivationTime;
     private int numStaleContainers;
+    private Instant lastLogTime;
 
     ContainerWatchdog() {
         this(new ScheduledThreadPoolExecutor(
@@ -50,8 +52,8 @@ class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable {
     ContainerWatchdog(ScheduledExecutorService scheduler, Clock clock) {
         this.scheduler = scheduler;
         this.clock = clock;
-        scheduler.scheduleAtFixedRate(
-                this::monitorDeactivatedContainers, UPDATE_PERIOD.getSeconds(), UPDATE_PERIOD.getSeconds(), TimeUnit.SECONDS);
+        this.lastLogTime = clock.instant();
+        scheduler.scheduleAtFixedRate(this::monitorDeactivatedContainers, CHECK_PERIOD.getSeconds(), CHECK_PERIOD.getSeconds(), TimeUnit.SECONDS);
     }
 
     @Override
@@ -77,31 +79,43 @@ class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable {
     void onContainerActivation(ActiveContainer nextContainer) {
         synchronized (monitor) {
             if (currentContainer != null) {
-                deactivatedContainers.add(
-                        new DeactivatedContainer(currentContainer, currentContainerActivationTime, clock.instant()));
+                deactivatedContainers.add(new DeactivatedContainer(currentContainer, currentContainerActivationTime, clock.instant()));
             }
             currentContainer = nextContainer;
             currentContainerActivationTime = clock.instant();
         }
     }
 
+    private String removalMsg(DeactivatedContainer container) {
+        return String.format("Removing deactivated container: instance=%s, activated=%s, deactivated=%s",
+                               container.instance, container.timeActivated, container.timeDeactivated);
+    }
+
+    private String regularMsg(DeactivatedContainer container, int refCount) {
+        return String.format(
+                "Deactivated container still alive: instance=%s, activated=%s, deactivated=%s, ref-count=%d",
+                container.instance, container.timeActivated, container.timeDeactivated, refCount);
+    }
+
     void monitorDeactivatedContainers() {
         synchronized (monitor) {
             int numStaleContainer = 0;
             Iterator<DeactivatedContainer> iterator = deactivatedContainers.iterator();
+            boolean timeToLogAgain = clock.instant().isAfter(lastLogTime.plus(UPDATE_PERIOD));
             while (iterator.hasNext()) {
                 DeactivatedContainer container = iterator.next();
                 int refCount = container.instance.retainCount();
                 if (refCount == 0) {
+                    log.fine(removalMsg(container));
                     iterator.remove();
-                    break;
-                }
-                if (isPastGracePeriod(container)) {
-                    ++numStaleContainer;
-                    log.warning(
-                            String.format(
-                                    "Deactivated container still alive: instance=%s, activated=%s, deactivated=%s, ref-count=%d",
-                                    container.instance.toString(), container.timeActivated, container.timeDeactivated, refCount));
+                } else {
+                    if (isPastGracePeriod(container)) {
+                        ++numStaleContainer;
+                        if (timeToLogAgain) {
+                            log.warning(regularMsg(container, refCount));
+                            lastLogTime = clock.instant();
+                        }
+                    }
                 }
             }
             this.numStaleContainers = numStaleContainer;
-- 
cgit v1.2.3