diff options
author | Bjørn Christian Seime <bjorncs@yahoo-inc.com> | 2017-06-07 16:13:40 +0200 |
---|---|---|
committer | Bjørn Christian Seime <bjorncs@yahoo-inc.com> | 2017-06-08 12:50:07 +0200 |
commit | eab973deb3dddf18fd3b806e04313a469c30a77d (patch) | |
tree | 8eb389d52e598135583b4171b34f228d79f66946 | |
parent | 025eae59e89a473dbff75b435df3dc4710b8a789 (diff) |
Use a single thread to monitor all deactivated ActiveContainers
Add new watchdog class to combine monitoring and reporting of stale
ActiveContainer instances. Introduce a grace period to allow deactivated
containers some time to die.
-rw-r--r-- | container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java | 14 | ||||
-rw-r--r-- | container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java | 4 | ||||
-rw-r--r-- | jdisc_core/pom.xml | 6 | ||||
-rw-r--r-- | jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java | 154 | ||||
-rw-r--r-- | jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java | 4 | ||||
-rw-r--r-- | jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java | 48 | ||||
-rw-r--r-- | jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerMetrics.java | 17 | ||||
-rw-r--r-- | jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerStatistics.java | 132 | ||||
-rw-r--r-- | jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java (renamed from jdisc_core/src/test/java/com/yahoo/jdisc/statistics/ActiveContainerStatisticsTest.java) | 48 |
9 files changed, 226 insertions, 201 deletions
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java index 4d71ae913d7..fafadc33d43 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java @@ -4,7 +4,7 @@ package com.yahoo.container.jdisc.metric; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.statistics.ActiveContainerStatistics; +import com.yahoo.jdisc.statistics.ActiveContainerMetrics; import java.nio.file.DirectoryStream; import java.nio.file.Files; @@ -32,19 +32,19 @@ public class MetricUpdater extends AbstractComponent { private static final String OPEN_FILE_DESCRIPTORS = "jdisc.open_file_descriptors"; private final Metric metric; - private final ActiveContainerStatistics activeContainerStatistics; + private final ActiveContainerMetrics activeContainerMetrics; private final Timer timer = new Timer(); long freeMemory = -1; long totalMemory = -1; @Inject - public MetricUpdater(Metric metric, ActiveContainerStatistics activeContainerStatistics) { - this(metric, activeContainerStatistics, 10*1000); + public MetricUpdater(Metric metric, ActiveContainerMetrics activeContainerMetrics) { + this(metric, activeContainerMetrics, 10*1000); } - public MetricUpdater(Metric metric, ActiveContainerStatistics activeContainerStatistics, long delayMillis) { + public MetricUpdater(Metric metric, ActiveContainerMetrics activeContainerMetrics, long delayMillis) { this.metric = metric; - this.activeContainerStatistics = activeContainerStatistics; + this.activeContainerMetrics = activeContainerMetrics; timer.schedule(new UpdaterTask(), delayMillis, delayMillis); } @@ -72,7 +72,7 @@ public class MetricUpdater extends AbstractComponent { metric.set(TOTAL_MEMORY_BYTES, totalMemory, null); metric.set(MEMORY_MAPPINGS_COUNT, count_mappings(), null); metric.set(OPEN_FILE_DESCRIPTORS, count_open_files(), null); - activeContainerStatistics.emitMetrics(metric); + activeContainerMetrics.emitMetrics(metric); } // Note: Linux-specific diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java index 1b88b5217bc..1ab0f8c090f 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java @@ -3,7 +3,7 @@ package com.yahoo.container.jdisc.metric; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.application.MetricConsumer; -import com.yahoo.jdisc.statistics.ActiveContainerStatistics; +import com.yahoo.jdisc.statistics.ActiveContainerMetrics; import org.junit.Test; import org.mockito.Mockito; @@ -17,7 +17,7 @@ public class MetricUpdaterTest { MetricProvider provider = MetricProviders.newInstance(consumer); Metric metric = provider.get(); - MetricUpdater updater = new MetricUpdater(metric, Mockito.mock(ActiveContainerStatistics.class), 10); + MetricUpdater updater = new MetricUpdater(metric, Mockito.mock(ActiveContainerMetrics.class), 10); long start = System.currentTimeMillis(); boolean updated = false; while (System.currentTimeMillis() - start < 60000 && !updated) { diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index bc6240919e9..b289d5816cd 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -31,6 +31,12 @@ <scope>test</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>aopalliance</groupId> <artifactId>aopalliance</artifactId> <scope>compile</scope> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java new file mode 100644 index 00000000000..7902a89b88e --- /dev/null +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java @@ -0,0 +1,154 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.core; + +import com.google.inject.Inject; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.statistics.ActiveContainerMetrics; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.WeakHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static java.util.Comparator.comparing; +import static java.util.stream.Collectors.toList; + +/** + * A watchdog that monitors all deactivated {@link ActiveContainer} instances with the purpose of detecting containers + * that are unable to be garbage collected by the JVM. + * + * @author bjorncs + */ +class ActiveContainerDeactivationWatchdog implements ActiveContainerMetrics, AutoCloseable { + static final Duration WATCHDOG_FREQUENCY = Duration.ofMinutes(20); + static final Duration ACTIVE_CONTAINER_GRACE_PERIOD = Duration.ofHours(1); + + private static final Logger log = Logger.getLogger(ActiveContainerDeactivationWatchdog.class.getName()); + + private final Object monitor = new Object(); + private final WeakHashMap<ActiveContainer, LifecycleStats> deactivatedContainers = new WeakHashMap<>(); + private final ScheduledExecutorService scheduler; + private final Clock clock; + + private ActiveContainer currentContainer; + private Instant currentContainerActivationTime; + + @Inject + ActiveContainerDeactivationWatchdog() { + this( + Clock.systemUTC(), + new ScheduledThreadPoolExecutor(1, runnable -> { + Thread thread = new Thread(runnable, "active-container-deactivation-watchdog"); + thread.setDaemon(true); + return thread; + })); + } + + ActiveContainerDeactivationWatchdog(Clock clock, ScheduledExecutorService scheduler) { + this.clock = clock; + this.scheduler = scheduler; + this.scheduler.scheduleWithFixedDelay( + this::warnOnStaleContainers, + WATCHDOG_FREQUENCY.getSeconds(), + WATCHDOG_FREQUENCY.getSeconds(), + TimeUnit.SECONDS); + } + + void onContainerActivation(ActiveContainer nextContainer) { + synchronized (monitor) { + Instant now = clock.instant(); + if (currentContainer != null) { + deactivatedContainers.put(currentContainer, new LifecycleStats(currentContainerActivationTime, now)); + } + currentContainer = nextContainer; + currentContainerActivationTime = now; + } + } + + @Override + public void emitMetrics(Metric metric) { + List<DeactivatedContainer> snapshot = getDeactivatedContainersSnapshot(); + long containersWithRetainedRefsCount = snapshot.stream() + .filter(c -> c.activeContainer.retainCount() > 0) + .count(); + metric.set(TOTAL_DEACTIVATED_CONTAINERS, snapshot.size(), null); + metric.set(DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES, containersWithRetainedRefsCount, null); + } + + @Override + public void close() { + synchronized (monitor) { + scheduler.shutdown(); + deactivatedContainers.clear(); + currentContainer = null; + currentContainerActivationTime = null; + } + } + + private void warnOnStaleContainers() { + try { + List<DeactivatedContainer> snapshot = getDeactivatedContainersSnapshot(); + if (snapshot.isEmpty()) return; + logWarning(snapshot); + } catch (Throwable t) { + log.log(Level.WARNING, "Watchdog task died!", t); + } + } + + private List<DeactivatedContainer> getDeactivatedContainersSnapshot() { + Instant now = clock.instant(); + synchronized (monitor) { + return deactivatedContainers.entrySet().stream() + .filter(e -> e.getValue().isPastGracePeriod(now)) + .map(e -> new DeactivatedContainer(e.getKey(), e.getValue())) + .sorted(comparing(e -> e.lifecycleStats.timeActivated)) + .collect(toList()); + } + } + + private static void logWarning(List<DeactivatedContainer> snapshot) { + log.warning(String.format("%s instances of deactivated containers are still alive.", snapshot.size())); + for (DeactivatedContainer deactivatedContainer : snapshot) { + log.warning(" - " + deactivatedContainer.toSummaryString()); + } + } + + private static class LifecycleStats { + public final Instant timeActivated; + public final Instant timeDeactivated; + + public LifecycleStats(Instant timeActivated, Instant timeDeactivated) { + this.timeActivated = timeActivated; + this.timeDeactivated = timeDeactivated; + } + + public boolean isPastGracePeriod(Instant instant) { + return timeDeactivated.plus(ACTIVE_CONTAINER_GRACE_PERIOD).isBefore(instant); + } + } + + private static class DeactivatedContainer { + public final ActiveContainer activeContainer; + public final LifecycleStats lifecycleStats; + + public DeactivatedContainer(ActiveContainer activeContainer, LifecycleStats lifecycleStats) { + this.activeContainer = activeContainer; + this.lifecycleStats = lifecycleStats; + } + + public String toSummaryString() { + return String.format("%s: time activated = %s, time deactivated = %s, reference count = %d", + activeContainer.toString(), + lifecycleStats.timeActivated.toString(), + lifecycleStats.timeDeactivated.toString(), + activeContainer.retainCount()); + } + } + +} diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java index c6acde814eb..956b2483ee0 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java @@ -8,7 +8,7 @@ import com.yahoo.jdisc.application.ContainerBuilder; import com.yahoo.jdisc.application.ContainerThread; import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.service.CurrentContainer; -import com.yahoo.jdisc.statistics.ActiveContainerStatistics; +import com.yahoo.jdisc.statistics.ActiveContainerMetrics; import java.util.concurrent.ThreadFactory; @@ -29,7 +29,7 @@ class ApplicationEnvironmentModule extends AbstractModule { bind(CurrentContainer.class).toInstance(loader); bind(OsgiFramework.class).toInstance(loader.osgiFramework()); bind(ThreadFactory.class).to(ContainerThread.Factory.class); - bind(ActiveContainerStatistics.class).toInstance(loader.getActiveContainerStatistics()); + bind(ActiveContainerMetrics.class).toInstance(loader.getActiveContainerMetrics()); } @Provides diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java index 4e63bc77c9a..8ddc62a6e19 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java @@ -15,23 +15,22 @@ import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.application.OsgiHeader; import com.yahoo.jdisc.service.ContainerNotReadyException; import com.yahoo.jdisc.service.CurrentContainer; -import com.yahoo.jdisc.statistics.ActiveContainerStatistics; +import com.yahoo.jdisc.statistics.ActiveContainerMetrics; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.BundleException; -import java.lang.ref.WeakReference; import java.net.URI; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; /** * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author bjorncs */ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, CurrentContainer { @@ -42,7 +41,7 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C private final AtomicReference<ActiveContainer> containerRef = new AtomicReference<>(); private final Object appLock = new Object(); private final List<Bundle> appBundles = new ArrayList<>(); - private final ActiveContainerStatistics statistics = new ActiveContainerStatistics(); + private final ActiveContainerDeactivationWatchdog watchdog = new ActiveContainerDeactivationWatchdog(); private Application application; private ApplicationInUseTracker applicationInUseTracker; @@ -71,46 +70,14 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C next.retainReference(applicationInUseTracker); } + watchdog.onContainerActivation(next); prev = containerRef.getAndSet(next); - statistics.onActivated(next); if (prev == null) { return null; } - statistics.onDeactivated(prev); } prev.release(); - DeactivatedContainer deactivatedContainer = prev.shutdown(); - - final WeakReference<ActiveContainer> prevContainerReference = new WeakReference<>(prev); - final Runnable deactivationMonitor = () -> { - long waitTimeSeconds = 30L; - long totalTimeWaited = 0L; - - while (!Thread.interrupted()) { - final long currentWaitTimeSeconds = waitTimeSeconds; - totalTimeWaited += currentWaitTimeSeconds; - - Interruption.mapExceptionToThreadState(() -> - Thread.sleep(TimeUnit.MILLISECONDS.convert(currentWaitTimeSeconds, TimeUnit.SECONDS)) - ); - - statistics.printSummaryToLog(); - final ActiveContainer prevContainer = prevContainerReference.get(); - if (prevContainer == null || prevContainer.retainCount() == 0) { - return; - } - log.warning("Previous container not terminated in the last " + totalTimeWaited + " seconds." - + " Reference state={ " + prevContainer.currentState() + " }"); - - waitTimeSeconds = (long) (waitTimeSeconds * 1.2); - } - log.warning("Deactivation monitor thread unexpectedly interrupted"); - }; - final Thread deactivationMonitorThread = new Thread(deactivationMonitor, "Container deactivation monitor"); - deactivationMonitorThread.setDaemon(true); - deactivationMonitorThread.start(); - - return deactivatedContainer; + return prev.shutdown(); } @Override @@ -228,6 +195,7 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C @Override public void destroy() { log.finer("Destroying application loader."); + watchdog.close(); try { osgiFramework.stop(); } catch (BundleException e) { @@ -241,8 +209,8 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C } } - public ActiveContainerStatistics getActiveContainerStatistics() { - return statistics; + public ActiveContainerMetrics getActiveContainerMetrics() { + return watchdog; } public OsgiFramework osgiFramework() { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerMetrics.java b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerMetrics.java new file mode 100644 index 00000000000..92ba6c5801a --- /dev/null +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerMetrics.java @@ -0,0 +1,17 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.statistics; + +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.core.ActiveContainer; + +/** + * Tracks statistics on stale {@link ActiveContainer} instances. + * + * @author bjorncs + */ +public interface ActiveContainerMetrics { + String TOTAL_DEACTIVATED_CONTAINERS = "jdisc.deactivated_containers.total"; + String DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES = "jdisc.deactivated_containers.with_retained_refs"; + + void emitMetrics(Metric metric); +} diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerStatistics.java b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerStatistics.java deleted file mode 100644 index 2fac3b4024b..00000000000 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerStatistics.java +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.statistics; - -import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.core.ActiveContainer; - -import java.time.Instant; -import java.util.List; -import java.util.WeakHashMap; -import java.util.logging.Logger; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toList; - -/** - * Tracks statistics on stale {@link ActiveContainer} instances. - * - * @author bjorncs - */ -public class ActiveContainerStatistics { - public interface Metrics { - String TOTAL_DEACTIVATED_CONTAINERS = "jdisc.deactivated_containers.total"; - String DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES = "jdisc.deactivated_containers.with_retained_refs"; - } - - private static final Logger log = Logger.getLogger(ActiveContainerStatistics.class.getName()); - - private final WeakHashMap<ActiveContainer, ActiveContainerStats> activeContainers = new WeakHashMap<>(); - private final Object monitor = new Object(); - - public void emitMetrics(Metric metric) { - synchronized (monitor) { - DeactivatedContainerMetrics metrics = deactivatedContainerStream() - .collect( - DeactivatedContainerMetrics::new, - DeactivatedContainerMetrics::aggregate, - DeactivatedContainerMetrics::merge); - - metric.set(Metrics.TOTAL_DEACTIVATED_CONTAINERS, metrics.deactivatedContainerCount, null); - metric.set(Metrics.DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES, metrics.deactivatedContainersWithRetainedRefsCount, null); - } - } - - public void onActivated(ActiveContainer activeContainer) { - synchronized (monitor) { - activeContainers.put(activeContainer, new ActiveContainerStats(Instant.now())); - } - } - - public void onDeactivated(ActiveContainer activeContainer) { - synchronized (monitor) { - ActiveContainerStats containerStats = activeContainers.get(activeContainer); - if (containerStats == null) { - throw new IllegalStateException("onActivated() has not been called for container: " + activeContainer); - } - containerStats.timeDeactivated = Instant.now(); - } - } - - public void printSummaryToLog() { - synchronized (monitor) { - List<DeactivatedContainer> deactivatedContainers = deactivatedContainerStream().collect(toList()); - if (deactivatedContainers.isEmpty()) return; - - log.warning( - "Multiple instances of ActiveContainer leaked! " + deactivatedContainers.size() + - " instances are still present."); - deactivatedContainers.stream() - .map(c -> " - " + c.toSummaryString()) - .forEach(log::warning); - } - } - - private Stream<DeactivatedContainer> deactivatedContainerStream() { - synchronized (monitor) { - return activeContainers.entrySet().stream() - .filter(e -> e.getValue().isDeactivated()) - .map(e -> new DeactivatedContainer(e.getKey(), e.getValue().timeActivated, e.getValue().timeDeactivated)); - } - } - - private static class ActiveContainerStats { - public final Instant timeActivated; - public Instant timeDeactivated; - - public ActiveContainerStats(Instant timeActivated) { - this.timeActivated = timeActivated; - } - - public boolean isDeactivated() { - return timeDeactivated != null; - } - } - - private static class DeactivatedContainer { - public final ActiveContainer activeContainer; - public final Instant timeActivated; - public final Instant timeDeactivated; - - public DeactivatedContainer(ActiveContainer activeContainer, Instant timeActivated, Instant timeDeactivated) { - this.activeContainer = activeContainer; - this.timeActivated = timeActivated; - this.timeDeactivated = timeDeactivated; - } - - public String toSummaryString() { - return String.format("%s: timeActivated=%s, timeDeactivated=%s, retainCount=%d", - activeContainer.toString(), - timeActivated.toString(), - timeDeactivated.toString(), - activeContainer.retainCount()); - } - } - - private static class DeactivatedContainerMetrics { - public int deactivatedContainerCount = 0; - public int deactivatedContainersWithRetainedRefsCount = 0; - - public void aggregate(DeactivatedContainer deactivatedContainer) { - ++deactivatedContainerCount; - if (deactivatedContainer.activeContainer.retainCount() > 0) { - ++deactivatedContainersWithRetainedRefsCount; - } - } - - public DeactivatedContainerMetrics merge(DeactivatedContainerMetrics other) { - deactivatedContainerCount += other.deactivatedContainerCount; - deactivatedContainersWithRetainedRefsCount += other.deactivatedContainersWithRetainedRefsCount; - return this; - } - } -} diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/statistics/ActiveContainerStatisticsTest.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java index 0a3482b705a..1a776f59e29 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/statistics/ActiveContainerStatisticsTest.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java @@ -1,48 +1,60 @@ -package com.yahoo.jdisc.statistics; +package com.yahoo.jdisc.core; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ResourceReference; -import com.yahoo.jdisc.core.ActiveContainer; +import com.yahoo.jdisc.statistics.ActiveContainerMetrics; import com.yahoo.jdisc.test.TestDriver; +import com.yahoo.test.ManualClock; import org.junit.Test; +import java.time.Duration; +import java.time.Instant; import java.util.Map; +import java.util.concurrent.Executors; import static org.junit.Assert.assertEquals; /** * @author bjorncs */ -public class ActiveContainerStatisticsTest { +public class ActiveContainerDeactivationWatchdogTest { @Test - public void counts_deactivated_activecontainers() { + public void watchdog_counts_deactivated_containers() { TestDriver driver = TestDriver.newSimpleApplicationInstanceWithoutOsgi(); - ActiveContainerStatistics stats = new ActiveContainerStatistics(); + ManualClock clock = new ManualClock(Instant.now()); + ActiveContainerDeactivationWatchdog watchdog = + new ActiveContainerDeactivationWatchdog(clock, Executors.newScheduledThreadPool(1)); MockMetric metric = new MockMetric(); ActiveContainer containerWithoutRetainedResources = new ActiveContainer(driver.newContainerBuilder()); - stats.onActivated(containerWithoutRetainedResources); - stats.emitMetrics(metric); + watchdog.onContainerActivation(containerWithoutRetainedResources); + watchdog.emitMetrics(metric); assertEquals(0, metric.totalCount); assertEquals(0, metric.withRetainedReferencesCount); - stats.onDeactivated(containerWithoutRetainedResources); + watchdog.onContainerActivation(null); containerWithoutRetainedResources.release(); - stats.emitMetrics(metric); + clock.advance(ActiveContainerDeactivationWatchdog.ACTIVE_CONTAINER_GRACE_PERIOD); + watchdog.emitMetrics(metric); + assertEquals(0, metric.totalCount); + assertEquals(0, metric.withRetainedReferencesCount); + + clock.advance(Duration.ofSeconds(1)); + watchdog.emitMetrics(metric); assertEquals(1, metric.totalCount); assertEquals(0, metric.withRetainedReferencesCount); ActiveContainer containerWithRetainedResources = new ActiveContainer(driver.newContainerBuilder()); - try (ResourceReference ignoredRef = containerWithRetainedResources.refer()) { - stats.onActivated(containerWithRetainedResources); - stats.onDeactivated(containerWithRetainedResources); - containerWithRetainedResources.release(); - stats.emitMetrics(metric); - assertEquals(2, metric.totalCount); - assertEquals(1, metric.withRetainedReferencesCount); + watchdog.onContainerActivation(containerWithRetainedResources); + containerWithRetainedResources.release(); + watchdog.onContainerActivation(null); + clock.advance(ActiveContainerDeactivationWatchdog.ACTIVE_CONTAINER_GRACE_PERIOD.plusSeconds(1)); + watchdog.emitMetrics(metric); + assertEquals(2, metric.totalCount); + assertEquals(1, metric.withRetainedReferencesCount); } } @@ -54,10 +66,10 @@ public class ActiveContainerStatisticsTest { @Override public void set(String key, Number val, Context ctx) { switch (key) { - case ActiveContainerStatistics.Metrics.TOTAL_DEACTIVATED_CONTAINERS: + case ActiveContainerMetrics.TOTAL_DEACTIVATED_CONTAINERS: totalCount = val.intValue(); break; - case ActiveContainerStatistics.Metrics.DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES: + case ActiveContainerMetrics.DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES: withRetainedReferencesCount = val.intValue(); break; default: |