Merge pull request #3088 from vespa-engine/balder/only-dump-stack-when-necessary

Balder/only dump stack when necessary

9 files changed, 59 insertions, 97 deletions

diff --git a/configd/src/apps/sentinel/command-connection.h b/configd/src/apps/sentinel/command-connection.h
index 532b70caf76..b637a519384 100644
--- a/configd/src/apps/sentinel/command-connection.h
+++ b/configd/src/apps/sentinel/command-connection.h
@@ -3,8 +3,7 @@
 
 #include "line-splitter.h"
 
-namespace config {
-namespace sentinel {
+namespace config::sentinel {
 
 class CommandConnection {
 private:
@@ -26,6 +25,4 @@ public:
     int fd() const { return _fd; }
 };
 
-} // end namespace sentinel
-} // end namespace config
-
+}
diff --git a/configd/src/apps/sentinel/config-handler.cpp b/configd/src/apps/sentinel/config-handler.cpp
index c9242370b66..de03f34c5ed 100644
--- a/configd/src/apps/sentinel/config-handler.cpp
+++ b/configd/src/apps/sentinel/config-handler.cpp
@@ -1,39 +1,22 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <sys/time.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <ctype.h>
-
-#include <list>
-#include <algorithm>
-
-
-#include <vespa/log/log.h>
-LOG_SETUP(".config-handler");
-LOG_RCSID("$Id$");
 
 #include "config-handler.h"
-#include "service.h"
 #include "command-connection.h"
 #include "output-connection.h"
 
 #include <vespa/vespalib/net/simple_metric_snapshot.h>
 #include <vespa/vespalib/net/socket_address.h>
 
-namespace config {
-namespace sentinel {
+#include <vespa/log/log.h>
+LOG_SETUP(".config-handler");
+
+namespace config::sentinel {
 
 int
 ConfigHandler::listen(int port) {
     auto handle = vespalib::SocketAddress::select_local(port).listen();
     if (!handle) {
-        LOG(error, "Fatal: listen on command control socket failed: %s",
-            strerror(errno));
+        LOG(error, "Fatal: listen on command control socket failed: %s", strerror(errno));
         EV_STOPPING("config-sentinel", "listen on command control socket failed");
         exit(EXIT_FAILURE);
     }
@@ -100,7 +83,7 @@ ConfigHandler::terminateServices(bool catchable, bool printDebug)
         if (printDebug && service->isRunning()) {
             LOG(info, "%s: killing", service->name().c_str());
         }
-        service->terminate(catchable);
+        service->terminate(catchable, printDebug);
     }
 }
 
@@ -538,7 +521,7 @@ ConfigHandler::doRestart(CommandConnection *c, char *args, bool force)
     }
     c->printf("terminating service %s pid %d, will be autorestarted\n",
               args, service->pid());
-    service->terminate(!force);
+    service->terminate(!force, false);
 }
 
 void
@@ -568,7 +551,7 @@ ConfigHandler::doStop(CommandConnection *c, char *args, bool force)
         return;
     }
     c->printf("Stopping %s.\n", args);
-    service->terminate(!force);
+    service->terminate(!force, false);
 }
 
 void
@@ -618,6 +601,4 @@ ConfigHandler::doManual(CommandConnection *c, char *args)
     }
 }
 
-
-} // end namespace config::sentinel
-} // end namespace config
+}
diff --git a/configd/src/apps/sentinel/config-handler.h b/configd/src/apps/sentinel/config-handler.h
index aa2fb3b5ea1..d294d97cff9 100644
--- a/configd/src/apps/sentinel/config-handler.h
+++ b/configd/src/apps/sentinel/config-handler.h
@@ -1,25 +1,22 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
+#include "service.h"
+#include "metrics.h"
+#include "state-api.h"
+#include <vespa/config-sentinel.h>
+#include <vespa/config/config.h>
+#include <vespa/vespalib/util/hashmap.h>
 #include <sys/types.h>
 #include <sys/select.h>
 
 #include <list>
-#include <vespa/vespalib/util/hashmap.h>
-
-#include <vespa/config-sentinel.h>
-#include <vespa/config/config.h>
-
-#include "service.h"
-#include "metrics.h"
-#include "state-api.h"
 
 using cloud::config::SentinelConfig;
 using config::ConfigSubscriber;
 using config::ConfigHandle;
 
-namespace config {
-namespace sentinel {
+namespace config::sentinel {
 
 class CommandConnection;
 class OutputConnection;
@@ -77,7 +74,4 @@ public:
     void updateActiveFdset(fd_set *fds, int *maxNum);
 };
 
-
-} // end namespace config::sentinel
-} // end namespace config
-
+}
diff --git a/configd/src/apps/sentinel/line-splitter.h b/configd/src/apps/sentinel/line-splitter.h
index 1fd9fce8653..f6802004ef7 100644
--- a/configd/src/apps/sentinel/line-splitter.h
+++ b/configd/src/apps/sentinel/line-splitter.h
@@ -1,8 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-namespace config {
-namespace sentinel {
+namespace config::sentinel {
 
 class LineSplitter {
 private:
@@ -28,6 +27,5 @@ public:
     ~LineSplitter();
 };
 
-} // end namespace config::sentinel
-} // end namespace config
+}
 
diff --git a/configd/src/apps/sentinel/metrics.h b/configd/src/apps/sentinel/metrics.h
index 87072410ef0..dd24bb2280d 100644
--- a/configd/src/apps/sentinel/metrics.h
+++ b/configd/src/apps/sentinel/metrics.h
@@ -3,8 +3,7 @@
 
 #include <sys/time.h>
 
-namespace config {
-namespace sentinel {
+namespace config::sentinel {
 
 struct StartMetrics {
     unsigned long currentlyRunningServices;
@@ -23,6 +22,5 @@ struct StartMetrics {
     void maybeLog();
 };
 
-} // end namespace config::sentinel
-} // end namespace config
+}
 
diff --git a/configd/src/apps/sentinel/output-connection.h b/configd/src/apps/sentinel/output-connection.h
index ff5a8c62bbb..fa8ef0e3270 100644
--- a/configd/src/apps/sentinel/output-connection.h
+++ b/configd/src/apps/sentinel/output-connection.h
@@ -5,8 +5,7 @@
 
 namespace ns_log { class LLParser; }
 
-namespace config {
-namespace sentinel {
+namespace config::sentinel {
 
 class OutputConnection {
 private:
@@ -27,6 +26,4 @@ public:
     int fd() const { return _fd; }
 };
 
-} // end namespace sentinel
-} // end namespace config
-
+}
diff --git a/configd/src/apps/sentinel/service.cpp b/configd/src/apps/sentinel/service.cpp
index 90a88931c14..492616c2db4 100644
--- a/configd/src/apps/sentinel/service.cpp
+++ b/configd/src/apps/sentinel/service.cpp
@@ -1,26 +1,28 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <sys/types.h>
+
+#include "service.h"
+#include "output-connection.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
 #include <signal.h>
-#include <errno.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <sys/wait.h>
 
 #include <vespa/log/log.h>
-LOG_SETUP(".service", "$Id$");
+LOG_SETUP(".service");
 #include <vespa/log/llparser.h>
 
-#include "service.h"
-#include "output-connection.h"
-
 extern sig_atomic_t stop;
 
+using vespalib::make_string;
+
 namespace config::sentinel {
 
 namespace {
 
-std::string getVespaTempDir() {
-    std::string tmp = getenv("ROOT");
+vespalib::string getVespaTempDir() {
+    vespalib::string tmp = getenv("ROOT");
     tmp += "/var/db/vespa/tmp";
     return tmp;
 }
@@ -71,7 +73,7 @@ Service::reconfigure(const SentinelConfig::Service& config)
     if (config.id != _config->id) {
         LOG(warning, "%s: reconfigured config id '%s' -> '%s' - signaling service restart",
             name().c_str(), _config->id.c_str(), config.id.c_str());
-        terminate(true);
+        terminate();
     }
 
     delete _config;
@@ -88,12 +90,12 @@ Service::reconfigure(const SentinelConfig::Service& config)
 
 Service::~Service()
 {
-    terminate(false);
+    terminate(false, false);
     delete _config;
 }
 
 int
-Service::terminate(bool catchable)
+Service::terminate(bool catchable, bool dumpState)
 {
     if (isRunning()) {
         runPreShutdownCommand();
@@ -107,12 +109,14 @@ Service::terminate(bool catchable)
             return ret;
         } else {
             setState(KILLING);
-            char pstackCmd[256];
-            LOG(info, "%s:%d failed to stop. Will dump the stack",  name().c_str(), _pid);
-            snprintf(pstackCmd, sizeof(pstackCmd), "pstack %d > %s/%s.pstack.%d", _pid, getVespaTempDir().c_str(), name().c_str(), _pid);
-            int pstackRet = system(pstackCmd);
-            if (pstackRet != 0) {
-                LOG(warning, "'%s' failed with return value %d", pstackCmd, pstackRet);
+            if (dumpState) {
+                vespalib::string pstackCmd = make_string("pstack %d > %s/%s.pstack.%d",
+                                                         _pid, getVespaTempDir().c_str(), name().c_str(), _pid);
+                LOG(info, "%s:%d failed to stop. Stack dumped at %s", name().c_str(), _pid, pstackCmd.c_str());
+                int pstackRet = system(pstackCmd.c_str());
+                if (pstackRet != 0) {
+                    LOG(warning, "'%s' failed with return value %d", pstackCmd.c_str(), pstackRet);
+                }
             }
             kill(_pid, SIGCONT); // if it was stopped for some reason
             int ret = kill(_pid, SIGKILL);
@@ -443,4 +447,3 @@ Service::stateName(ServiceState state) const
 }
 
 }
-
diff --git a/configd/src/apps/sentinel/service.h b/configd/src/apps/sentinel/service.h
index d6bdbadd8f6..6419fdd7268 100644
--- a/configd/src/apps/sentinel/service.h
+++ b/configd/src/apps/sentinel/service.h
@@ -1,16 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
+#include "metrics.h"
 #include <vespa/vespalib/stllike/string.h>
 #include <vespa/config-sentinel.h>
 #include <list>
 
-#include "metrics.h"
-
 using cloud::config::SentinelConfig;
 
-namespace config {
-namespace sentinel {
+namespace config::sentinel {
 
 class OutputConnection;
 
@@ -49,11 +47,14 @@ public:
     ~Service();
     Service(const SentinelConfig::Service& config,
             const SentinelConfig::Application& application,
-	    std::list<OutputConnection *> &ocs,
-            StartMetrics &metrics);
+			std::list<OutputConnection *> &ocs,
+			StartMetrics &metrics);
     void reconfigure(const SentinelConfig::Service& config);
     int pid() const { return _pid; }
-    int terminate(bool catchable);
+    int terminate(bool catchable, bool dumpState);
+    int terminate() {
+        return terminate(true, false);
+    }
     int start();
     void youExited(int status); // Call this if waitpid says it exited
     const vespalib::string & name() const;
@@ -67,6 +68,4 @@ public:
     void incrementRestartPenalty();
 };
 
-} // end namespace sentinel
-} // end namespace config
-
+}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp
index c8e00e70ee7..fc0a57ff519 100644
--- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp
+++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp
@@ -1,16 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "fdispatch.h" 
-
-#include <vespa/searchcore/util/log.h>
-#include <vespa/searchcore/util/eventloop.h>
+#include "engineadapter.h"
+#include "rpc.h"
 #include <vespa/searchcore/fdispatch/search/querycacheutil.h>
 #include <vespa/searchcore/fdispatch/search/nodemanager.h>
+#include <vespa/searchcore/util/eventloop.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/config/helper/configgetter.hpp>
-#include "engineadapter.h"
-#include "rpc.h"
-#include <thread>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".fdispatch");
@@ -259,9 +256,7 @@ void Fdispatch::configure(std::unique_ptr<FdispatchrcConfig> cfg)
 {
     if (cfg && _config) {
         if ( needRestart(*_config, *cfg) ) {
-            const int sleepMS = (0.100 + 2 * _rndGen.nextDouble()) * 1000;
-            LOG(warning, "Will restart by abort in %d ms.", sleepMS);
-            std::this_thread::sleep_for(std::chrono::milliseconds(sleepMS));
+            LOG(warning, "Will restart by abort now.");
             _needRestart.store(true);
         }
     }