change splunkd startup logic

* doing "start" then "restart" seems to trigger splunkd core dumps,
  so we will try "stop" + wait + "start" instead.
* log logforwarder actions on info level to make it easier to
  track splunk activity.

1 files changed, 10 insertions, 9 deletions

diff --git a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp
index 0588ce5d2e7..c5c19aa2c3f 100644
--- a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp
+++ b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp
@@ -27,7 +27,7 @@ runSplunk(const vespalib::string &prefix, std::vector<const char *> args)
         dbg.append(arg);
         dbg.append("'");
     }
-    LOG(debug, "starting splunk forwarder with command: %s", dbg.c_str());
+    LOG(info, "trigger splunk with command: %s", dbg.c_str());
     args.push_back(nullptr);
     fflush(stdout);
     pid_t child = fork();
@@ -58,13 +58,13 @@ runSplunk(const vespalib::string &prefix, std::vector<const char *> args)
         return;
     }
     if (WIFEXITED(waitStatus)) {
-        LOG(warning, "failed starting splunk forwarder (exit status %d)",
+        LOG(warning, "failed triggering splunk (exit status %d)",
             WEXITSTATUS(waitStatus));
     } else if (WIFSIGNALED(waitStatus)) {
-        LOG(warning, "failed starting splunk forwarder (exit on signal %d)",
+        LOG(warning, "failed triggering splunk (exit on signal %d)",
             WTERMSIG(waitStatus));
     } else {
-        LOG(warning, "failed starting splunk forwarder (abnormal exit status %d)",
+        LOG(warning, "failed triggering splunk (abnormal exit status %d)",
             waitStatus);
     }
 }
@@ -76,14 +76,15 @@ void
 ChildHandler::startChild(const vespalib::string &prefix)
 {
     if (! _childRunning) {
+        // it is possible that splunk was already running anyway, so
+        // make sure we restart it to get new config activated:
+        runSplunk(prefix, {"stop"});
+        sleep(1);
         runSplunk(prefix, {"start", "--answer-yes", "--no-prompt", "--accept-license"});
         _childRunning = true;
-        // it is possible that splunk was already running, and
-        // then the above won't do anything, so we need to
-        // *also* do the restart below, after a small delay.
-        sleep(1);
+    } else {
+        runSplunk(prefix, {"restart"});
     }
-    runSplunk(prefix, {"restart"});
 }
 
 void