Dump heap of live objects on failed application shutdown.

3 files changed, 27 insertions, 3 deletions

diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
index 9fe3728dc2c..be97c06749a 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
@@ -40,8 +40,10 @@ import com.yahoo.log.LogSetup;
 import com.yahoo.messagebus.network.rpc.SlobrokConfigSubscriber;
 import com.yahoo.net.HostName;
 import com.yahoo.vespa.config.ConfigKey;
+import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.yolean.Exceptions;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
@@ -400,9 +402,17 @@ public final class ConfiguredApplication implements Application {
         shutdownDeadlineExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("Shutdown deadline timer"));
         shutdownDeadlineExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
         long delayMillis = 50 * 1000;
-        shutdownDeadlineExecutor.schedule(() -> com.yahoo.protect.Process.logAndDie(
-                "Timed out waiting for application shutdown. Please check that all your request handlers " +
-                        "drain their request content channels.", true), delayMillis, TimeUnit.MILLISECONDS);
+        shutdownDeadlineExecutor.schedule(() -> {
+            String heapDumpName = Defaults.getDefaults().underVespaHome("tmp/java_shutdown.") + ProcessHandle.current().pid() + ".hprof";
+            try {
+                com.yahoo.protect.Process.dumpHeap(heapDumpName, true);
+            } catch (IOException e) {
+                log.log(Level.WARNING, "Failed writing heap dump:", e);
+            }
+            com.yahoo.protect.Process.logAndDie(
+                    "Timed out waiting for application shutdown. Please check that all your request handlers " +
+                            "drain their request content channels.", true);
+        }, delayMillis, TimeUnit.MILLISECONDS);
     }
 
     private static void addHandlerBindings(ContainerBuilder builder,
diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json
index 199291ac276..e5d5b8ba5b6 100644
--- a/vespajlib/abi-spec.json
+++ b/vespajlib/abi-spec.json
@@ -658,6 +658,7 @@
       "public static void logAndDie(java.lang.String, boolean)",
       "public static void logAndDie(java.lang.String, java.lang.Throwable)",
       "public static void logAndDie(java.lang.String, java.lang.Throwable, boolean)",
+      "public static void dumpHeap(java.lang.String, boolean)",
       "public static void dumpThreads()"
     ],
     "fields": []
diff --git a/vespajlib/src/main/java/com/yahoo/protect/Process.java b/vespajlib/src/main/java/com/yahoo/protect/Process.java
index 4d2fafd4665..bc5f0d8f876 100644
--- a/vespajlib/src/main/java/com/yahoo/protect/Process.java
+++ b/vespajlib/src/main/java/com/yahoo/protect/Process.java
@@ -1,7 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.protect;
 
+import com.sun.management.HotSpotDiagnosticMXBean;
 
+import javax.management.MBeanServer;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Level;
@@ -70,6 +74,15 @@ public final class Process {
         }
     }
 
+    public static void dumpHeap(String filePath, boolean live) throws IOException {
+        getHotspotMXBean().dumpHeap(filePath, live);
+    }
+
+    private static HotSpotDiagnosticMXBean getHotspotMXBean() throws IOException {
+        MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+        return ManagementFactory.newPlatformMXBeanProxy(
+                server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class);
+    }
 
     public static void dumpThreads() {
         boolean alreadyDumpingThreads = busyDumpingThreads.getAndSet(true);