Remove container watchdog

- Remove ActiveContainerDeactivationWatchdog
- Move deconstruction enforcer to ActiveContainer

7 files changed, 20 insertions, 443 deletions

diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java
index 907d2a050fb..38fa91b4831 100644
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java
@@ -18,12 +18,16 @@ import com.yahoo.jdisc.service.ServerProvider;
 import java.net.URI;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.logging.Logger;
 
 /**
  * @author Simon Thoresen
+ * @author bjorncs
  */
 public class ActiveContainer extends AbstractResource implements CurrentContainer {
 
+    private static final Logger log = Logger.getLogger(ActiveContainer.class.getName());
+
     private final ContainerTermination termination;
     private final Injector guiceInjector;
     private final Iterable<ServerProvider> serverProviders;
@@ -32,7 +36,7 @@ public class ActiveContainer extends AbstractResource implements CurrentContaine
     private final Map<String, BindingSet<RequestHandler>> clientBindings;
     private final BindingSetSelector bindingSetSelector;
     private final TimeoutManagerImpl timeoutMgr;
-    final Destructor destructor;
+    private final Destructor destructor;
 
     public ActiveContainer(ContainerBuilder builder) {
         serverProviders = builder.serverProviders().activate();
@@ -122,8 +126,19 @@ public class ActiveContainer extends AbstractResource implements CurrentContaine
         return new ContainerSnapshot(this, serverBindings, clientBindings);
     }
 
+    // TODO Rewrite to use cleaners after Java 9 migration
+    @Override
+    protected void finalize() throws Throwable {
+        boolean alreadyDestructed = destructor.destruct();
+        if (!alreadyDestructed) {
+            log.severe(toString() + " was not correctly cleaned up " +
+                               "because of a resource leak or invalid use of reference counting.");
+        }
+        super.finalize();
+    }
+
     // NOTE: An instance of this class must never contain a reference to the outer class (ActiveContainer).
-    static class Destructor {
+    private static class Destructor {
         private final ResourcePool resourceReferences;
         private final TimeoutManagerImpl timeoutMgr;
         private final ContainerTermination termination;
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java
deleted file mode 100644
index e41c149c0af..00000000000
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java
+++ /dev/null
@@ -1,232 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.jdisc.core;
-
-import com.google.inject.Inject;
-import com.yahoo.jdisc.Metric;
-import com.yahoo.jdisc.statistics.ActiveContainerMetrics;
-
-import java.lang.ref.PhantomReference;
-import java.lang.ref.ReferenceQueue;
-import java.time.Clock;
-import java.time.Duration;
-import java.time.Instant;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.WeakHashMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import static java.util.Comparator.comparing;
-import static java.util.stream.Collectors.toList;
-
-/**
- * A watchdog that monitors all deactivated {@link ActiveContainer} instances with the purpose of detecting containers
- * that are unable to be garbage collected by the JVM.
- *
- * @author bjorncs
- */
-// TODO Rewrite to use cleaners instead of phantom references after Java 9 migration
-class ActiveContainerDeactivationWatchdog implements ActiveContainerMetrics, AutoCloseable {
-    // Frequency of writing warnings on stale container in the log
-    static final Duration REPORTING_FREQUENCY = Duration.ofMinutes(20);
-    // Only stale containers past the report grace period will be included in the metrics and log reporting
-    static final Duration REPORTING_GRACE_PERIOD = Duration.ofHours(4);
-
-    // The frequency specifies how often gc can be triggered.
-    // This value should be a fraction of REPORTING_GRACE_PERIOD to eliminate chance of reporting deactivated containers
-    // that are unreferenced, but have not been gc'ed yet.
-    static final Duration GC_TRIGGER_FREQUENCY = Duration.ofHours(1); // Must be a fraction of REPORTING_GRACE_PERIOD
-    // Gc will only be triggered if there are any stale containers past the grace period.
-    // This value should be a fraction of REPORTING_GRACE_PERIOD to eliminate chance of reporting deactivated containers
-    // that are unreferenced, but have not been gc'ed yet.
-    static final Duration GC_GRACE_PERIOD = Duration.ofHours(2);
-
-    // How often destruction of stale containers should be performed
-    static final Duration ENFORCE_DESTRUCTION_GCED_CONTAINERS_FREQUENCY = Duration.ofMinutes(5);
-
-    private static final Logger log = Logger.getLogger(ActiveContainerDeactivationWatchdog.class.getName());
-
-    private final Object monitor = new Object();
-    private final WeakHashMap<ActiveContainer, LifecycleStats> deactivatedContainers = new WeakHashMap<>();
-    private final ReferenceQueue<ActiveContainer> garbageCollectedContainers = new ReferenceQueue<>();
-    @SuppressWarnings("MismatchedQueryAndUpdateOfCollection")
-    // Instances of the phantom references must be kept alive until they are polled from the reference queue
-    private final Set<ActiveContainerPhantomReference> destructorReferences = new HashSet<>();
-    private final ScheduledExecutorService scheduler;
-    private final Clock clock;
-
-    private ActiveContainer currentContainer;
-    private Instant currentContainerActivationTime;
-
-    @Inject
-    ActiveContainerDeactivationWatchdog() {
-        this(
-                Clock.systemUTC(),
-                new ScheduledThreadPoolExecutor(1, runnable -> {
-                    Thread thread = new Thread(runnable, "active-container-deactivation-watchdog");
-                    thread.setDaemon(true);
-                    return thread;
-                }));
-    }
-
-    ActiveContainerDeactivationWatchdog(Clock clock,
-                                        ScheduledExecutorService scheduler) {
-        this.clock = clock;
-        this.scheduler = scheduler;
-        // NOTE: Make sure to update the unit test if the order commands are registered is changed.
-        this.scheduler.scheduleAtFixedRate(this::warnOnStaleContainers,
-                                           REPORTING_FREQUENCY.getSeconds(),
-                                           REPORTING_FREQUENCY.getSeconds(),
-                                           TimeUnit.SECONDS);
-        this.scheduler.scheduleAtFixedRate(this::triggerGc,
-                                           GC_TRIGGER_FREQUENCY.getSeconds(),
-                                           GC_TRIGGER_FREQUENCY.getSeconds(),
-                                           TimeUnit.SECONDS);
-        this.scheduler.scheduleAtFixedRate(this::enforceDestructionOfGarbageCollectedContainers,
-                                           ENFORCE_DESTRUCTION_GCED_CONTAINERS_FREQUENCY.getSeconds(),
-                                           ENFORCE_DESTRUCTION_GCED_CONTAINERS_FREQUENCY.getSeconds(),
-                                           TimeUnit.SECONDS);
-    }
-
-    void onContainerActivation(ActiveContainer nextContainer) {
-        synchronized (monitor) {
-            Instant now = clock.instant();
-            if (currentContainer != null) {
-                deactivatedContainers.put(currentContainer, new LifecycleStats(currentContainerActivationTime, now));
-                destructorReferences.add(new ActiveContainerPhantomReference(currentContainer, garbageCollectedContainers));
-            }
-            currentContainer = nextContainer;
-            currentContainerActivationTime = now;
-        }
-    }
-
-    @Override
-    public void emitMetrics(Metric metric) {
-        List<DeactivatedContainer> snapshot = getDeactivatedContainersSnapshot(REPORTING_GRACE_PERIOD);
-        long containersWithRetainedRefsCount = snapshot.stream()
-                .filter(c -> c.activeContainer.retainCount() > 0)
-                .count();
-        metric.set(TOTAL_DEACTIVATED_CONTAINERS, snapshot.size(), null);
-        metric.set(DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES, containersWithRetainedRefsCount, null);
-    }
-
-    @Override
-    public void close() {
-        synchronized (monitor) {
-            scheduler.shutdown();
-            deactivatedContainers.clear();
-            destructorReferences.clear();
-            currentContainer = null;
-            currentContainerActivationTime = null;
-        }
-    }
-
-    private void warnOnStaleContainers() {
-        log.log(Level.FINE, "Checking for stale containers");
-        try {
-            List<DeactivatedContainer> snapshot = getDeactivatedContainersSnapshot(REPORTING_GRACE_PERIOD);
-            if (snapshot.isEmpty()) return;
-            logWarning(snapshot);
-        } catch (Throwable t) {
-            log.log(Level.WARNING, "Watchdog task died!", t);
-        }
-    }
-
-    private void triggerGc() {
-        int deactivatedContainers = getDeactivatedContainersSnapshot(GC_GRACE_PERIOD).size();
-        boolean shouldGc = deactivatedContainers > 0;
-        if (!shouldGc) return;
-        log.log(Level.INFO, String.format("Triggering GC and finalization (currently %d deactivated containers still alive)", deactivatedContainers));
-        System.gc();
-        System.runFinalization();
-    }
-
-    private void enforceDestructionOfGarbageCollectedContainers() {
-        log.log(Level.FINE, "Enforcing destruction of GCed containers");
-        ActiveContainerPhantomReference reference;
-        while ((reference = (ActiveContainerPhantomReference) garbageCollectedContainers.poll()) != null) {
-            try {
-                reference.enforceDestruction();
-            } catch (Throwable t) {
-                log.log(Level.SEVERE, "Failed to do post-GC destruction of " + reference.containerName, t);
-            } finally {
-                destructorReferences.remove(reference);
-                reference.clear();
-            }
-        }
-    }
-
-    private List<DeactivatedContainer> getDeactivatedContainersSnapshot(Duration gracePeriod) {
-        Instant now = clock.instant();
-        synchronized (monitor) {
-            return deactivatedContainers.entrySet().stream()
-                    .filter(e -> e.getValue().isPastGracePeriod(now, gracePeriod))
-                    .map(e -> new DeactivatedContainer(e.getKey(), e.getValue()))
-                    .sorted(comparing(e -> e.lifecycleStats.timeActivated))
-                    .collect(toList());
-        }
-    }
-
-    private static void logWarning(List<DeactivatedContainer> snapshot) {
-        log.warning(String.format("%s instances of deactivated containers are still alive.", snapshot.size()));
-        for (DeactivatedContainer deactivatedContainer : snapshot) {
-            log.warning(" - " + deactivatedContainer.toSummaryString());
-        }
-    }
-
-    private static class LifecycleStats {
-        public final Instant timeActivated;
-        public final Instant timeDeactivated;
-
-        public LifecycleStats(Instant timeActivated, Instant timeDeactivated) {
-            this.timeActivated = timeActivated;
-            this.timeDeactivated = timeDeactivated;
-        }
-
-        public boolean isPastGracePeriod(Instant instant, Duration gracePeriod) {
-            return timeDeactivated.plus(gracePeriod).isBefore(instant);
-        }
-    }
-
-    private static class DeactivatedContainer {
-        public final ActiveContainer activeContainer;
-        public final LifecycleStats lifecycleStats;
-
-        public DeactivatedContainer(ActiveContainer activeContainer, LifecycleStats lifecycleStats) {
-            this.activeContainer = activeContainer;
-            this.lifecycleStats = lifecycleStats;
-        }
-
-        public String toSummaryString() {
-            return String.format("%s: time activated = %s, time deactivated = %s, reference count = %d",
-                    activeContainer.toString(),
-                    lifecycleStats.timeActivated.toString(),
-                    lifecycleStats.timeDeactivated.toString(),
-                    activeContainer.retainCount());
-        }
-    }
-
-    private static class ActiveContainerPhantomReference extends PhantomReference<ActiveContainer> {
-        public final String containerName;
-        private final ActiveContainer.Destructor destructor;
-
-        public ActiveContainerPhantomReference(ActiveContainer activeContainer,
-                                               ReferenceQueue<? super ActiveContainer> q) {
-            super(activeContainer, q);
-            this.containerName = activeContainer.toString();
-            this.destructor = activeContainer.destructor;
-        }
-
-        public void enforceDestruction() {
-            boolean alreadyCompleted = destructor.destruct();
-            if (!alreadyCompleted) {
-                log.severe(containerName + " was not correctly cleaned up " +
-                        "because of a resource leak or invalid use of reference counting.");
-            }
-        }
-    }
-}
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java
index 7d2f300ab7d..b7cab923454 100644
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.jdisc.core;
 
 import com.google.inject.AbstractModule;
@@ -8,12 +8,11 @@ import com.yahoo.jdisc.application.ContainerBuilder;
 import com.yahoo.jdisc.application.ContainerThread;
 import com.yahoo.jdisc.application.OsgiFramework;
 import com.yahoo.jdisc.service.CurrentContainer;
-import com.yahoo.jdisc.statistics.ActiveContainerMetrics;
 
 import java.util.concurrent.ThreadFactory;
 
 /**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @author Simon Thoresen
  */
 class ApplicationEnvironmentModule extends AbstractModule {
 
@@ -29,7 +28,6 @@ class ApplicationEnvironmentModule extends AbstractModule {
         bind(CurrentContainer.class).toInstance(loader);
         bind(OsgiFramework.class).toInstance(loader.osgiFramework());
         bind(ThreadFactory.class).to(ContainerThread.Factory.class);
-        bind(ActiveContainerMetrics.class).toInstance(loader.getActiveContainerMetrics());
     }
 
     @Provides
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java
index 10b4c9f6508..81eb5815a01 100644
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.jdisc.core;
 
 import com.google.inject.AbstractModule;
@@ -15,7 +15,6 @@ import com.yahoo.jdisc.application.OsgiFramework;
 import com.yahoo.jdisc.application.OsgiHeader;
 import com.yahoo.jdisc.service.ContainerNotReadyException;
 import com.yahoo.jdisc.service.CurrentContainer;
-import com.yahoo.jdisc.statistics.ActiveContainerMetrics;
 import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.BundleException;
@@ -41,7 +40,6 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C
     private final AtomicReference<ActiveContainer> containerRef = new AtomicReference<>();
     private final Object appLock = new Object();
     private final List<Bundle> appBundles = new ArrayList<>();
-    private final ActiveContainerDeactivationWatchdog watchdog = new ActiveContainerDeactivationWatchdog();
     private Application application;
     private ApplicationInUseTracker applicationInUseTracker;
 
@@ -70,7 +68,6 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C
                 next.retainReference(applicationInUseTracker);
             }
 
-            watchdog.onContainerActivation(next);
             prev = containerRef.getAndSet(next);
             if (prev == null) {
                 return null;
@@ -195,7 +192,6 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C
     @Override
     public void destroy() {
         log.finer("Destroying application loader.");
-        watchdog.close();
         try {
             osgiFramework.stop();
         } catch (BundleException e) {
@@ -209,10 +205,6 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C
         }
     }
 
-    public ActiveContainerMetrics getActiveContainerMetrics() {
-        return watchdog;
-    }
-
     public OsgiFramework osgiFramework() {
         return osgiFramework;
     }
@@ -227,18 +219,4 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C
         }
     }
 
-    private static class Interruption {
-        interface Runnable_throws<E extends Throwable>  {
-            void run() throws E;
-        }
-
-        public static void mapExceptionToThreadState(Runnable_throws<InterruptedException> runnable) {
-            try {
-                runnable.run();
-            } catch (InterruptedException ignored) {
-                Thread.currentThread().interrupt();
-            }
-        }
-    }
-
 }
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerMetrics.java b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerMetrics.java
deleted file mode 100644
index 440c9af6bf5..00000000000
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerMetrics.java
+++ /dev/null
@@ -1,19 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.jdisc.statistics;
-
-import com.yahoo.jdisc.Metric;
-import com.yahoo.jdisc.core.ActiveContainer;
-
-/**
- * Tracks statistics on stale {@link ActiveContainer} instances.
- *
- * @author bjorncs
- */
-public interface ActiveContainerMetrics {
-
-    String TOTAL_DEACTIVATED_CONTAINERS = "jdisc.deactivated_containers.total";
-    String DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES = "jdisc.deactivated_containers.with_retained_refs";
-
-    void emitMetrics(Metric metric);
-
-}
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/package-info.java b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/package-info.java
deleted file mode 100644
index 2558e693e39..00000000000
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/package-info.java
+++ /dev/null
@@ -1,4 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-@com.yahoo.osgi.annotation.ExportPackage
-package com.yahoo.jdisc.statistics;
diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java
deleted file mode 100644
index e2de08e760f..00000000000
--- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java
+++ /dev/null
@@ -1,159 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.jdisc.core;
-
-import com.yahoo.jdisc.Metric;
-import com.yahoo.jdisc.ResourceReference;
-import com.yahoo.jdisc.statistics.ActiveContainerMetrics;
-import com.yahoo.jdisc.test.TestDriver;
-import com.yahoo.test.ManualClock;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.lang.ref.WeakReference;
-import java.time.Duration;
-import java.time.Instant;
-import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author bjorncs
- */
-public class ActiveContainerDeactivationWatchdogTest {
-
-    @Test
-    public void watchdog_counts_deactivated_containers() {
-        TestDriver driver = TestDriver.newSimpleApplicationInstanceWithoutOsgi();
-        ManualClock clock = new ManualClock(Instant.now());
-        ActiveContainerDeactivationWatchdog watchdog =
-                new ActiveContainerDeactivationWatchdog(clock, Executors.newScheduledThreadPool(1));
-        MockMetric metric = new MockMetric();
-
-        ActiveContainer containerWithoutRetainedResources = new ActiveContainer(driver.newContainerBuilder());
-
-        watchdog.onContainerActivation(containerWithoutRetainedResources);
-        watchdog.emitMetrics(metric);
-        assertEquals(0, metric.totalCount);
-        assertEquals(0, metric.withRetainedReferencesCount);
-
-        watchdog.onContainerActivation(null);
-        containerWithoutRetainedResources.release();
-        clock.advance(ActiveContainerDeactivationWatchdog.REPORTING_GRACE_PERIOD);
-        watchdog.emitMetrics(metric);
-        assertEquals(0, metric.totalCount);
-        assertEquals(0, metric.withRetainedReferencesCount);
-
-        clock.advance(Duration.ofSeconds(1));
-        watchdog.emitMetrics(metric);
-        assertEquals(1, metric.totalCount);
-        assertEquals(0, metric.withRetainedReferencesCount);
-
-        ActiveContainer containerWithRetainedResources = new ActiveContainer(driver.newContainerBuilder());
-        try (ResourceReference ignoredRef = containerWithRetainedResources.refer()) {
-            watchdog.onContainerActivation(containerWithRetainedResources);
-            containerWithRetainedResources.release();
-            watchdog.onContainerActivation(null);
-            clock.advance(ActiveContainerDeactivationWatchdog.REPORTING_GRACE_PERIOD.plusSeconds(1));
-            watchdog.emitMetrics(metric);
-            assertEquals(2, metric.totalCount);
-            assertEquals(1, metric.withRetainedReferencesCount);
-        }
-
-    }
-
-    @Test
-    @Ignore("Ignored as it assumes phantom references are enqueued right after first GC have cleared the weak reference. " +
-            "This is the case on most JVMs.")
-    public void deactivated_container_destructed_if_its_reference_counter_is_nonzero() {
-        ExecutorMock executor = new ExecutorMock();
-        ManualClock clock = new ManualClock(Instant.now());
-        ActiveContainerDeactivationWatchdog watchdog = new ActiveContainerDeactivationWatchdog(clock, executor);
-        ActiveContainer container =
-                new ActiveContainer(TestDriver.newSimpleApplicationInstanceWithoutOsgi().newContainerBuilder());
-        AtomicBoolean destructed = new AtomicBoolean(false);
-        container.shutdown().notifyTermination(() -> destructed.set(true));
-
-        container.refer(); // increase reference counter to simulate a leaking resource
-        watchdog.onContainerActivation(container);
-        container.release(); // release resource
-        watchdog.onContainerActivation(null); // deactivate container
-
-        WeakReference<ActiveContainer> containerWeakReference = new WeakReference<>(container);
-        container = null; // make container instance collectable by GC
-        clock.advance(ActiveContainerDeactivationWatchdog.GC_GRACE_PERIOD.plusSeconds(1));
-
-        executor.triggerGcCommand.run();
-
-        assertNull("Container is not GCed - probably because the watchdog has a concrete reference to it",
-                   containerWeakReference.get());
-
-        executor.enforceDestructionOfGarbageCollectedContainersCommand.run();
-
-        assertTrue("Destructor is not called on deactivated container", destructed.get());
-    }
-
-    private static class MockMetric implements Metric {
-        public int totalCount;
-        public int withRetainedReferencesCount;
-
-        @Override
-        public void set(String key, Number val, Context ctx) {
-            switch (key) {
-                case ActiveContainerMetrics.TOTAL_DEACTIVATED_CONTAINERS:
-                    totalCount = val.intValue();
-                    break;
-                case ActiveContainerMetrics.DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES:
-                    withRetainedReferencesCount = val.intValue();
-                    break;
-                default:
-                    throw new UnsupportedOperationException();
-            }
-        }
-
-        @Override
-        public void add(String key, Number val, Context ctx) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public Context createContext(Map<String, ?> properties) {
-            throw new UnsupportedOperationException();
-        }
-    }
-
-    private static class ExecutorMock extends ScheduledThreadPoolExecutor {
-
-        public Runnable warnOnStaleContainersCommand;
-        public Runnable triggerGcCommand;
-        public Runnable enforceDestructionOfGarbageCollectedContainersCommand;
-        private int registrationCounter = 0;
-
-        public ExecutorMock() {
-            super(1);
-        }
-
-        @Override
-        public ScheduledFuture<?> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) {
-            if (registrationCounter == 0) {
-                warnOnStaleContainersCommand = command;
-            } else if (registrationCounter == 1) {
-                triggerGcCommand = command;
-            } else if (registrationCounter == 2) {
-                enforceDestructionOfGarbageCollectedContainersCommand = command;
-            } else {
-                throw new IllegalStateException("Unexpected registration");
-            }
-            ++registrationCounter;
-            return null;
-        }
-
-    }
-
-}