diff options
author | Bjørn Christian Seime <bjorncs@verizonmedia.com> | 2022-02-07 14:52:49 +0100 |
---|---|---|
committer | Bjørn Christian Seime <bjorncs@verizonmedia.com> | 2022-02-07 14:52:49 +0100 |
commit | 5732785a8c5612c46c20c4c24d1f375ebb3e2e7f (patch) | |
tree | a81c8c3c83e243ba98031e0508e37564725ef3d4 /container-disc/src | |
parent | 3d13b2df6632929fba8fa63fd3a712190898d5cd (diff) |
Reapply "Shutdown reconfiguration thread in a more controlled way [run-systemtest]""
Diffstat (limited to 'container-disc/src')
-rw-r--r-- | container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java | 85 |
1 files changed, 45 insertions, 40 deletions
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 24dcf18b555..64e94bd5f03 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -94,14 +94,15 @@ public final class ConfiguredApplication implements Application { new ComponentRegistry<>()); private final OsgiFramework restrictedOsgiFramework; private final Phaser nonTerminatedContainerTracker = new Phaser(1); + private final Thread reconfigurerThread; + private final Thread portWatcher; private HandlersConfigurerDi configurer; - private Thread reconfigurerThread; - private Thread portWatcher; private QrConfig qrConfig; private Register slobrokRegistrator = null; private Supervisor supervisor = null; private Acceptor acceptor = null; + private volatile boolean shutdownReconfiguration = false; static { LogSetup.initVespaLogging("Container"); @@ -135,6 +136,8 @@ public final class ConfiguredApplication implements Application { : Optional.empty(); this.restrictedOsgiFramework = new DisableOsgiFramework(new RestrictedBundleContext(osgiFramework.bundleContext())); this.shutdownDeadline = new ShutdownDeadline(configId); + this.reconfigurerThread = new Thread(this::doReconfigurationLoop, "configured-application-reconfigurer"); + this.portWatcher = new Thread(this::watchPortChange, "configured-application-port-watcher"); } @Override @@ -146,8 +149,9 @@ public final class ConfiguredApplication implements Application { ContainerBuilder builder = createBuilderWithGuiceBindings(); configurer = createConfigurer(builder.guiceModules().activate()); initializeAndActivateContainer(builder, () -> {}); - startReconfigurerThread(); - portWatcher = new Thread(this::watchPortChange, "configured-application-port-watcher"); + reconfigurerThread.setDaemon(true); + reconfigurerThread.start(); + portWatcher.setDaemon(true); portWatcher.start(); if (setupRpc()) { @@ -286,30 +290,27 @@ public final class ConfiguredApplication implements Application { } @SuppressWarnings("removal") // TODO Vespa 8: remove - private void startReconfigurerThread() { - reconfigurerThread = new Thread(() -> { - while ( ! Thread.interrupted()) { - try { - ContainerBuilder builder = createBuilderWithGuiceBindings(); - - // Block until new config arrives, and it should be applied - Runnable cleanupTask = configurer.waitForNextGraphGeneration(builder.guiceModules().activate(), false); - initializeAndActivateContainer(builder, cleanupTask); - } catch (UncheckedInterruptedException | ConfigInterruptedException e) { - break; - } catch (Exception | LinkageError e) { // LinkageError: OSGi problems - tryReportFailedComponentGraphConstructionMetric(configurer, e); - log.log(Level.SEVERE, - "Reconfiguration failed, your application package must be fixed, unless this is a " + - "JNI reload issue: " + Exceptions.toMessageString(e), e); - } catch (Error e) { - com.yahoo.protect.Process.logAndDie("java.lang.Error on reconfiguration: We are probably in " + - "a bad state and will terminate", e); - } + private void doReconfigurationLoop() { + while (!shutdownReconfiguration) { + try { + ContainerBuilder builder = createBuilderWithGuiceBindings(); + + // Block until new config arrives, and it should be applied + Runnable cleanupTask = configurer.waitForNextGraphGeneration(builder.guiceModules().activate(), false); + initializeAndActivateContainer(builder, cleanupTask); + } catch (UncheckedInterruptedException | ConfigInterruptedException e) { + break; + } catch (Exception | LinkageError e) { // LinkageError: OSGi problems + tryReportFailedComponentGraphConstructionMetric(configurer, e); + log.log(Level.SEVERE, + "Reconfiguration failed, your application package must be fixed, unless this is a " + + "JNI reload issue: " + Exceptions.toMessageString(e), e); + } catch (Error e) { + com.yahoo.protect.Process.logAndDie("java.lang.Error on reconfiguration: We are probably in " + + "a bad state and will terminate", e); } - log.fine("Shutting down HandlersConfigurerDi"); - }, "configured-application-reconfigurer"); - reconfigurerThread.start(); + } + log.fine("Reconfiguration loop exited"); } private static void tryReportFailedComponentGraphConstructionMetric(HandlersConfigurerDi configurer, Throwable error) { @@ -397,7 +398,7 @@ public final class ConfiguredApplication implements Application { } private void stopServersAndAwaitTermination(String logPrefix) { - shutdownReconfigurerThread(); + shutdownReconfigurer(); log.info(logPrefix + ": Closing servers"); startAndStopServers(List.of()); startAndRemoveClients(List.of()); @@ -407,20 +408,24 @@ public final class ConfiguredApplication implements Application { log.info(logPrefix + ": Finished"); } - // TODO Do more graceful shutdown of reconfigurer thread. The interrupt may leave the container in state where - // graceful shutdown is impossible or may hang. - private void shutdownReconfigurerThread() { + private void shutdownReconfigurer() { + if (!reconfigurerThread.isAlive()) { + log.info("Reconfiguration thread shutdown already completed"); + return; + } + log.info("Shutting down reconfiguration thread"); + long start = System.currentTimeMillis(); + shutdownReconfiguration = true; + configurer.shutdownConfigRetriever(); try { - //Workaround for component constructors masking InterruptedException. - while (reconfigurerThread != null && reconfigurerThread.isAlive()) { - reconfigurerThread.interrupt(); - long millis = 200; - reconfigurerThread.join(millis); - reconfigurerThread = null; - } + reconfigurerThread.join(); + log.info(String.format( + "Reconfiguration shutdown completed in %.3f seconds", (System.currentTimeMillis() - start) / 1000D)); } catch (InterruptedException e) { - log.info("Interrupted while joining on HandlersConfigurer reconfigure thread."); - Thread.currentThread().interrupt(); + String message = "Interrupted while waiting for reconfiguration shutdown"; + log.warning(message); + log.log(Level.FINE, e.getMessage(), e); + throw new UncheckedInterruptedException(message, true); } } |