Use a single thread to monitor all deactivated ActiveContainers

Add new watchdog class to combine monitoring and reporting of stale
ActiveContainer instances. Introduce a grace period to allow deactivated
containers some time to die.

7 files changed, 217 insertions, 192 deletions

diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml
index bc6240919e9..b289d5816cd 100644
--- a/jdisc_core/pom.xml
+++ b/jdisc_core/pom.xml
@@ -31,6 +31,12 @@
             <scope>test</scope>
         </dependency>
         <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>testutil</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
             <groupId>aopalliance</groupId>
             <artifactId>aopalliance</artifactId>
             <scope>compile</scope>
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java
new file mode 100644
index 00000000000..7902a89b88e
--- /dev/null
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java
@@ -0,0 +1,154 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.jdisc.core;
+
+import com.google.inject.Inject;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.statistics.ActiveContainerMetrics;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.WeakHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static java.util.Comparator.comparing;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * A watchdog that monitors all deactivated {@link ActiveContainer} instances with the purpose of detecting containers
+ * that are unable to be garbage collected by the JVM.
+ *
+ * @author bjorncs
+ */
+class ActiveContainerDeactivationWatchdog implements ActiveContainerMetrics, AutoCloseable {
+    static final Duration WATCHDOG_FREQUENCY = Duration.ofMinutes(20);
+    static final Duration ACTIVE_CONTAINER_GRACE_PERIOD = Duration.ofHours(1);
+
+    private static final Logger log = Logger.getLogger(ActiveContainerDeactivationWatchdog.class.getName());
+
+    private final Object monitor = new Object();
+    private final WeakHashMap<ActiveContainer, LifecycleStats> deactivatedContainers = new WeakHashMap<>();
+    private final ScheduledExecutorService scheduler;
+    private final Clock clock;
+
+    private ActiveContainer currentContainer;
+    private Instant currentContainerActivationTime;
+
+    @Inject
+    ActiveContainerDeactivationWatchdog() {
+        this(
+                Clock.systemUTC(),
+                new ScheduledThreadPoolExecutor(1, runnable -> {
+                    Thread thread = new Thread(runnable, "active-container-deactivation-watchdog");
+                    thread.setDaemon(true);
+                    return thread;
+                }));
+    }
+
+    ActiveContainerDeactivationWatchdog(Clock clock, ScheduledExecutorService scheduler) {
+        this.clock = clock;
+        this.scheduler = scheduler;
+        this.scheduler.scheduleWithFixedDelay(
+                this::warnOnStaleContainers,
+                WATCHDOG_FREQUENCY.getSeconds(),
+                WATCHDOG_FREQUENCY.getSeconds(),
+                TimeUnit.SECONDS);
+    }
+
+    void onContainerActivation(ActiveContainer nextContainer) {
+        synchronized (monitor) {
+            Instant now = clock.instant();
+            if (currentContainer != null) {
+                deactivatedContainers.put(currentContainer, new LifecycleStats(currentContainerActivationTime, now));
+            }
+            currentContainer = nextContainer;
+            currentContainerActivationTime = now;
+        }
+    }
+
+    @Override
+    public void emitMetrics(Metric metric) {
+        List<DeactivatedContainer> snapshot = getDeactivatedContainersSnapshot();
+        long containersWithRetainedRefsCount = snapshot.stream()
+                .filter(c -> c.activeContainer.retainCount() > 0)
+                .count();
+        metric.set(TOTAL_DEACTIVATED_CONTAINERS, snapshot.size(), null);
+        metric.set(DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES, containersWithRetainedRefsCount, null);
+    }
+
+    @Override
+    public void close() {
+        synchronized (monitor) {
+            scheduler.shutdown();
+            deactivatedContainers.clear();
+            currentContainer = null;
+            currentContainerActivationTime = null;
+        }
+    }
+
+    private void warnOnStaleContainers() {
+        try {
+            List<DeactivatedContainer> snapshot = getDeactivatedContainersSnapshot();
+            if (snapshot.isEmpty()) return;
+            logWarning(snapshot);
+        } catch (Throwable t) {
+            log.log(Level.WARNING, "Watchdog task died!", t);
+        }
+    }
+
+    private List<DeactivatedContainer> getDeactivatedContainersSnapshot() {
+        Instant now = clock.instant();
+        synchronized (monitor) {
+            return deactivatedContainers.entrySet().stream()
+                    .filter(e -> e.getValue().isPastGracePeriod(now))
+                    .map(e -> new DeactivatedContainer(e.getKey(), e.getValue()))
+                    .sorted(comparing(e -> e.lifecycleStats.timeActivated))
+                    .collect(toList());
+        }
+    }
+
+    private static void logWarning(List<DeactivatedContainer> snapshot) {
+        log.warning(String.format("%s instances of deactivated containers are still alive.", snapshot.size()));
+        for (DeactivatedContainer deactivatedContainer : snapshot) {
+            log.warning(" - " + deactivatedContainer.toSummaryString());
+        }
+    }
+
+    private static class LifecycleStats {
+        public final Instant timeActivated;
+        public final Instant timeDeactivated;
+
+        public LifecycleStats(Instant timeActivated, Instant timeDeactivated) {
+            this.timeActivated = timeActivated;
+            this.timeDeactivated = timeDeactivated;
+        }
+
+        public boolean isPastGracePeriod(Instant instant) {
+            return timeDeactivated.plus(ACTIVE_CONTAINER_GRACE_PERIOD).isBefore(instant);
+        }
+    }
+
+    private static class DeactivatedContainer {
+        public final ActiveContainer activeContainer;
+        public final LifecycleStats lifecycleStats;
+
+        public DeactivatedContainer(ActiveContainer activeContainer, LifecycleStats lifecycleStats) {
+            this.activeContainer = activeContainer;
+            this.lifecycleStats = lifecycleStats;
+        }
+
+        public String toSummaryString() {
+            return String.format("%s: time activated = %s, time deactivated = %s, reference count = %d",
+                    activeContainer.toString(),
+                    lifecycleStats.timeActivated.toString(),
+                    lifecycleStats.timeDeactivated.toString(),
+                    activeContainer.retainCount());
+        }
+    }
+
+}
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java
index c6acde814eb..956b2483ee0 100644
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java
@@ -8,7 +8,7 @@ import com.yahoo.jdisc.application.ContainerBuilder;
 import com.yahoo.jdisc.application.ContainerThread;
 import com.yahoo.jdisc.application.OsgiFramework;
 import com.yahoo.jdisc.service.CurrentContainer;
-import com.yahoo.jdisc.statistics.ActiveContainerStatistics;
+import com.yahoo.jdisc.statistics.ActiveContainerMetrics;
 
 import java.util.concurrent.ThreadFactory;
 
@@ -29,7 +29,7 @@ class ApplicationEnvironmentModule extends AbstractModule {
         bind(CurrentContainer.class).toInstance(loader);
         bind(OsgiFramework.class).toInstance(loader.osgiFramework());
         bind(ThreadFactory.class).to(ContainerThread.Factory.class);
-        bind(ActiveContainerStatistics.class).toInstance(loader.getActiveContainerStatistics());
+        bind(ActiveContainerMetrics.class).toInstance(loader.getActiveContainerMetrics());
     }
 
     @Provides
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java
index 4e63bc77c9a..8ddc62a6e19 100644
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java
@@ -15,23 +15,22 @@ import com.yahoo.jdisc.application.OsgiFramework;
 import com.yahoo.jdisc.application.OsgiHeader;
 import com.yahoo.jdisc.service.ContainerNotReadyException;
 import com.yahoo.jdisc.service.CurrentContainer;
-import com.yahoo.jdisc.statistics.ActiveContainerStatistics;
+import com.yahoo.jdisc.statistics.ActiveContainerMetrics;
 import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.BundleException;
 
-import java.lang.ref.WeakReference;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
  * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @author bjorncs
  */
 public class ApplicationLoader implements BootstrapLoader, ContainerActivator, CurrentContainer {
 
@@ -42,7 +41,7 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C
     private final AtomicReference<ActiveContainer> containerRef = new AtomicReference<>();
     private final Object appLock = new Object();
     private final List<Bundle> appBundles = new ArrayList<>();
-    private final ActiveContainerStatistics statistics = new ActiveContainerStatistics();
+    private final ActiveContainerDeactivationWatchdog watchdog = new ActiveContainerDeactivationWatchdog();
     private Application application;
     private ApplicationInUseTracker applicationInUseTracker;
 
@@ -71,46 +70,14 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C
                 next.retainReference(applicationInUseTracker);
             }
 
+            watchdog.onContainerActivation(next);
             prev = containerRef.getAndSet(next);
-            statistics.onActivated(next);
             if (prev == null) {
                 return null;
             }
-            statistics.onDeactivated(prev);
         }
         prev.release();
-        DeactivatedContainer deactivatedContainer = prev.shutdown();
-
-        final WeakReference<ActiveContainer> prevContainerReference = new WeakReference<>(prev);
-        final Runnable deactivationMonitor = () -> {
-            long waitTimeSeconds = 30L;
-            long totalTimeWaited = 0L;
-
-            while (!Thread.interrupted()) {
-                final long currentWaitTimeSeconds = waitTimeSeconds;
-                totalTimeWaited += currentWaitTimeSeconds;
-
-                Interruption.mapExceptionToThreadState(() ->
-                                Thread.sleep(TimeUnit.MILLISECONDS.convert(currentWaitTimeSeconds, TimeUnit.SECONDS))
-                );
-
-                statistics.printSummaryToLog();
-                final ActiveContainer prevContainer = prevContainerReference.get();
-                if (prevContainer == null || prevContainer.retainCount() == 0) {
-                    return;
-                }
-                log.warning("Previous container not terminated in the last " + totalTimeWaited + " seconds."
-                        + " Reference state={ " + prevContainer.currentState() + " }");
-
-                waitTimeSeconds = (long) (waitTimeSeconds * 1.2);
-            }
-            log.warning("Deactivation monitor thread unexpectedly interrupted");
-        };
-        final Thread deactivationMonitorThread = new Thread(deactivationMonitor, "Container deactivation monitor");
-        deactivationMonitorThread.setDaemon(true);
-        deactivationMonitorThread.start();
-
-        return deactivatedContainer;
+        return prev.shutdown();
     }
 
     @Override
@@ -228,6 +195,7 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C
     @Override
     public void destroy() {
         log.finer("Destroying application loader.");
+        watchdog.close();
         try {
             osgiFramework.stop();
         } catch (BundleException e) {
@@ -241,8 +209,8 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C
         }
     }
 
-    public ActiveContainerStatistics getActiveContainerStatistics() {
-        return statistics;
+    public ActiveContainerMetrics getActiveContainerMetrics() {
+        return watchdog;
     }
 
     public OsgiFramework osgiFramework() {
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerMetrics.java b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerMetrics.java
new file mode 100644
index 00000000000..92ba6c5801a
--- /dev/null
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerMetrics.java
@@ -0,0 +1,17 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.jdisc.statistics;
+
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.core.ActiveContainer;
+
+/**
+ * Tracks statistics on stale {@link ActiveContainer} instances.
+ *
+ * @author bjorncs
+ */
+public interface ActiveContainerMetrics {
+    String TOTAL_DEACTIVATED_CONTAINERS = "jdisc.deactivated_containers.total";
+    String DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES = "jdisc.deactivated_containers.with_retained_refs";
+
+    void emitMetrics(Metric metric);
+}
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerStatistics.java b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerStatistics.java
deleted file mode 100644
index 2fac3b4024b..00000000000
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerStatistics.java
+++ /dev/null
@@ -1,132 +0,0 @@
-// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.jdisc.statistics;
-
-import com.yahoo.jdisc.Metric;
-import com.yahoo.jdisc.core.ActiveContainer;
-
-import java.time.Instant;
-import java.util.List;
-import java.util.WeakHashMap;
-import java.util.logging.Logger;
-import java.util.stream.Stream;
-
-import static java.util.stream.Collectors.toList;
-
-/**
- * Tracks statistics on stale {@link ActiveContainer} instances.
- *
- * @author bjorncs
- */
-public class ActiveContainerStatistics {
-    public interface Metrics {
-        String TOTAL_DEACTIVATED_CONTAINERS = "jdisc.deactivated_containers.total";
-        String DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES = "jdisc.deactivated_containers.with_retained_refs";
-    }
-
-    private static final Logger log = Logger.getLogger(ActiveContainerStatistics.class.getName());
-
-    private final WeakHashMap<ActiveContainer, ActiveContainerStats> activeContainers = new WeakHashMap<>();
-    private final Object monitor = new Object();
-
-    public void emitMetrics(Metric metric) {
-        synchronized (monitor) {
-            DeactivatedContainerMetrics metrics = deactivatedContainerStream()
-                    .collect(
-                            DeactivatedContainerMetrics::new,
-                            DeactivatedContainerMetrics::aggregate,
-                            DeactivatedContainerMetrics::merge);
-
-            metric.set(Metrics.TOTAL_DEACTIVATED_CONTAINERS, metrics.deactivatedContainerCount, null);
-            metric.set(Metrics.DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES, metrics.deactivatedContainersWithRetainedRefsCount, null);
-        }
-    }
-
-    public void onActivated(ActiveContainer activeContainer) {
-        synchronized (monitor) {
-            activeContainers.put(activeContainer, new ActiveContainerStats(Instant.now()));
-        }
-    }
-
-    public void onDeactivated(ActiveContainer activeContainer) {
-        synchronized (monitor) {
-            ActiveContainerStats containerStats = activeContainers.get(activeContainer);
-            if (containerStats == null) {
-                throw new IllegalStateException("onActivated() has not been called for container: " + activeContainer);
-            }
-            containerStats.timeDeactivated = Instant.now();
-        }
-    }
-
-    public void printSummaryToLog() {
-        synchronized (monitor) {
-            List<DeactivatedContainer> deactivatedContainers = deactivatedContainerStream().collect(toList());
-            if (deactivatedContainers.isEmpty()) return;
-
-            log.warning(
-                    "Multiple instances of ActiveContainer leaked! " + deactivatedContainers.size() +
-                            " instances are still present.");
-            deactivatedContainers.stream()
-                    .map(c -> " - " + c.toSummaryString())
-                    .forEach(log::warning);
-        }
-    }
-
-    private Stream<DeactivatedContainer> deactivatedContainerStream() {
-        synchronized (monitor) {
-            return activeContainers.entrySet().stream()
-                    .filter(e -> e.getValue().isDeactivated())
-                    .map(e -> new DeactivatedContainer(e.getKey(), e.getValue().timeActivated, e.getValue().timeDeactivated));
-        }
-    }
-
-    private static class ActiveContainerStats {
-        public final Instant timeActivated;
-        public Instant timeDeactivated;
-
-        public ActiveContainerStats(Instant timeActivated) {
-            this.timeActivated = timeActivated;
-        }
-
-        public boolean isDeactivated() {
-            return timeDeactivated != null;
-        }
-    }
-
-    private static class DeactivatedContainer {
-        public final ActiveContainer activeContainer;
-        public final Instant timeActivated;
-        public final Instant timeDeactivated;
-
-        public DeactivatedContainer(ActiveContainer activeContainer, Instant timeActivated, Instant timeDeactivated) {
-            this.activeContainer = activeContainer;
-            this.timeActivated = timeActivated;
-            this.timeDeactivated = timeDeactivated;
-        }
-
-        public String toSummaryString() {
-            return String.format("%s: timeActivated=%s, timeDeactivated=%s, retainCount=%d",
-                    activeContainer.toString(),
-                    timeActivated.toString(),
-                    timeDeactivated.toString(),
-                    activeContainer.retainCount());
-        }
-    }
-
-    private static class DeactivatedContainerMetrics {
-        public int deactivatedContainerCount = 0;
-        public int deactivatedContainersWithRetainedRefsCount = 0;
-
-        public void aggregate(DeactivatedContainer deactivatedContainer) {
-            ++deactivatedContainerCount;
-            if (deactivatedContainer.activeContainer.retainCount() > 0) {
-                ++deactivatedContainersWithRetainedRefsCount;
-            }
-        }
-
-        public DeactivatedContainerMetrics merge(DeactivatedContainerMetrics other) {
-            deactivatedContainerCount += other.deactivatedContainerCount;
-            deactivatedContainersWithRetainedRefsCount += other.deactivatedContainersWithRetainedRefsCount;
-            return this;
-        }
-    }
-}
diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/statistics/ActiveContainerStatisticsTest.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java
index 0a3482b705a..1a776f59e29 100644
--- a/jdisc_core/src/test/java/com/yahoo/jdisc/statistics/ActiveContainerStatisticsTest.java
+++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java
@@ -1,48 +1,60 @@
-package com.yahoo.jdisc.statistics;
+package com.yahoo.jdisc.core;
 
 import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.ResourceReference;
-import com.yahoo.jdisc.core.ActiveContainer;
+import com.yahoo.jdisc.statistics.ActiveContainerMetrics;
 import com.yahoo.jdisc.test.TestDriver;
+import com.yahoo.test.ManualClock;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Map;
+import java.util.concurrent.Executors;
 
 import static org.junit.Assert.assertEquals;
 
 /**
  * @author bjorncs
  */
-public class ActiveContainerStatisticsTest {
+public class ActiveContainerDeactivationWatchdogTest {
 
     @Test
-    public void counts_deactivated_activecontainers() {
+    public void watchdog_counts_deactivated_containers() {
         TestDriver driver = TestDriver.newSimpleApplicationInstanceWithoutOsgi();
-        ActiveContainerStatistics stats = new ActiveContainerStatistics();
+        ManualClock clock = new ManualClock(Instant.now());
+        ActiveContainerDeactivationWatchdog watchdog =
+                new ActiveContainerDeactivationWatchdog(clock, Executors.newScheduledThreadPool(1));
         MockMetric metric = new MockMetric();
 
         ActiveContainer containerWithoutRetainedResources = new ActiveContainer(driver.newContainerBuilder());
 
-        stats.onActivated(containerWithoutRetainedResources);
-        stats.emitMetrics(metric);
+        watchdog.onContainerActivation(containerWithoutRetainedResources);
+        watchdog.emitMetrics(metric);
         assertEquals(0, metric.totalCount);
         assertEquals(0, metric.withRetainedReferencesCount);
 
-        stats.onDeactivated(containerWithoutRetainedResources);
+        watchdog.onContainerActivation(null);
         containerWithoutRetainedResources.release();
-        stats.emitMetrics(metric);
+        clock.advance(ActiveContainerDeactivationWatchdog.ACTIVE_CONTAINER_GRACE_PERIOD);
+        watchdog.emitMetrics(metric);
+        assertEquals(0, metric.totalCount);
+        assertEquals(0, metric.withRetainedReferencesCount);
+
+        clock.advance(Duration.ofSeconds(1));
+        watchdog.emitMetrics(metric);
         assertEquals(1, metric.totalCount);
         assertEquals(0, metric.withRetainedReferencesCount);
 
         ActiveContainer containerWithRetainedResources = new ActiveContainer(driver.newContainerBuilder());
-
         try (ResourceReference ignoredRef = containerWithRetainedResources.refer()) {
-             stats.onActivated(containerWithRetainedResources);
-             stats.onDeactivated(containerWithRetainedResources);
-             containerWithRetainedResources.release();
-             stats.emitMetrics(metric);
-             assertEquals(2, metric.totalCount);
-             assertEquals(1, metric.withRetainedReferencesCount);
+            watchdog.onContainerActivation(containerWithRetainedResources);
+            containerWithRetainedResources.release();
+            watchdog.onContainerActivation(null);
+            clock.advance(ActiveContainerDeactivationWatchdog.ACTIVE_CONTAINER_GRACE_PERIOD.plusSeconds(1));
+            watchdog.emitMetrics(metric);
+            assertEquals(2, metric.totalCount);
+            assertEquals(1, metric.withRetainedReferencesCount);
         }
 
     }
@@ -54,10 +66,10 @@ public class ActiveContainerStatisticsTest {
         @Override
         public void set(String key, Number val, Context ctx) {
             switch (key) {
-                case ActiveContainerStatistics.Metrics.TOTAL_DEACTIVATED_CONTAINERS:
+                case ActiveContainerMetrics.TOTAL_DEACTIVATED_CONTAINERS:
                     totalCount = val.intValue();
                     break;
-                case ActiveContainerStatistics.Metrics.DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES:
+                case ActiveContainerMetrics.DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES:
                     withRetainedReferencesCount = val.intValue();
                     break;
                 default: