6 files changed, 400 insertions, 0 deletions

diff --git a/client/go/script-utils/services/configproxy.go b/client/go/script-utils/services/configproxy.go
new file mode 100644
index 00000000000..fb452a4de00
--- /dev/null
+++ b/client/go/script-utils/services/configproxy.go
@@ -0,0 +1,134 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/vespa-engine/vespa/client/go/defaults"
+	"github.com/vespa-engine/vespa/client/go/envvars"
+	"github.com/vespa-engine/vespa/client/go/jvm"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+const (
+	CONFIGPROXY_PIDFILE = "var/run/configproxy.pid"
+	PROXY_SERVICE_NAME  = "configproxy"
+)
+
+func JustRunConfigproxy() int {
+	commonPreChecks()
+	vespa.CheckCorrectUser()
+	configsources := defaults.VespaConfigserverRpcAddrs()
+	if len(configsources) < 1 {
+		util.JustExitMsg("could not find any configservers")
+	}
+	util.TuneResourceLimits()
+	c := jvm.NewConfigProxyJvm(PROXY_SERVICE_NAME)
+	userargs := os.Getenv(envvars.VESPA_CONFIGPROXY_JVMARGS)
+	c.ConfigureOptions(configsources, userargs)
+	c.Exec()
+	// unreachable:
+	return 1
+}
+
+func startProxyWithRunserver() {
+	commonPreChecks()
+	vespa.CheckCorrectUser()
+	configsources := defaults.VespaConfigserverRpcAddrs()
+	fmt.Printf(
+		"Starting config proxy using %s as config source(s)\n",
+		strings.Join(configsources, " and "))
+	args := []string{
+		"-r", "10",
+		"-s", PROXY_SERVICE_NAME,
+		"-p", CONFIGPROXY_PIDFILE,
+		"--",
+		"libexec/vespa/script-utils", "just-run-configproxy",
+	}
+	cmd := exec.Command("vespa-runserver", args...)
+	cmd.Stdin = nil
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Start()
+	p := cmd.Process
+	if p != nil {
+		p.Release()
+	}
+}
+
+func waitForProxyResponse() bool {
+	hname, _ := vespa.FindOurHostname()
+	backtick := util.BackTicksWithStderr
+	start := time.Now()
+	fmt.Printf("Waiting for config proxy to start\n")
+	for sleepcount := 0; sleepcount < 1800; sleepcount++ {
+		time.Sleep(100 * time.Millisecond)
+		got, err := os.ReadFile(CONFIGPROXY_PIDFILE)
+		if err == nil {
+			pid, err := strconv.Atoi(strings.TrimSpace(string(got)))
+			if err == nil && pid > 0 {
+				out, err := backtick.Run("vespa-ping-configproxy", "-s", hname)
+				if err == nil {
+					secs := time.Since(start).Seconds()
+					fmt.Printf("config proxy started after %ds (runserver pid %d)\n", int(secs), pid)
+					return true
+				}
+				if sleepcount%50 == 9 {
+					trace.Warning("Could not ping configproxy:", err)
+					if sleepcount%500 == 59 {
+						secs := time.Since(start).Seconds()
+						trace.Warning("ping output after", int(secs), "seconds:", strings.TrimSpace(out))
+						logFile := defaults.VespaLogFile()
+						cmd := fmt.Sprintf("tail -n 15 %s | vespa-logfmt -l all -N", logFile)
+						out, err = backtick.Run("sh", "-c", cmd)
+						fmt.Fprintf(os.Stderr, "tail of logfile: >>>\n%s<<<\n", out)
+					}
+				}
+			} else {
+				trace.Debug("bad contents (", string(got), ") in pid file", CONFIGPROXY_PIDFILE)
+			}
+		} else {
+			trace.Debug("bad pid file", CONFIGPROXY_PIDFILE, err)
+		}
+	}
+	secs := time.Since(start).Seconds()
+	fmt.Fprintf(os.Stderr, "Config proxy still failed to start after %d seconds!\n", int(secs))
+	got, err := os.ReadFile(CONFIGPROXY_PIDFILE)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "pid file %s was not created\n", CONFIGPROXY_PIDFILE)
+		return false
+	}
+	gotpid := strings.TrimSpace(string(got))
+	pid, err := strconv.Atoi(gotpid)
+	if err != nil || pid < 1 {
+		fmt.Fprintf(os.Stderr, "invalid pid '%s' in file %s\n", gotpid, CONFIGPROXY_PIDFILE)
+		return false
+	}
+	out, err := backtick.Run("kill", "-0", gotpid)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "config proxy process `%s` has terminated: %s\n", gotpid, strings.TrimSpace(out))
+		return false
+	}
+	out, err = backtick.Run("vespa-ping-configproxy", "-s", hname)
+	fmt.Fprintf(os.Stderr, "failed to ping configproxy: %s\n", out)
+	return false
+}
+
+func StartConfigproxy() int {
+	commonPreChecks()
+	vespa.MaybeSwitchUser("start-configproxy")
+	startProxyWithRunserver()
+	if waitForProxyResponse() {
+		return 0
+	}
+	return 1
+}
diff --git a/client/go/script-utils/services/env.go b/client/go/script-utils/services/env.go
new file mode 100644
index 00000000000..1a51d1a21c7
--- /dev/null
+++ b/client/go/script-utils/services/env.go
@@ -0,0 +1,25 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/vespa-engine/vespa/client/go/envvars"
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+func exportSettings(vespaHome string) {
+	vlt := fmt.Sprintf("file:%s/logs/vespa/vespa.log", vespaHome)
+	lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome)
+	dlp := fmt.Sprintf("%s/lib64:/opt/vespa-deps/lib64", vespaHome)
+	os.Setenv(envvars.VESPA_LOG_TARGET, vlt)
+	os.Setenv(envvars.VESPA_LOG_CONTROL_DIR, lcd)
+	os.Setenv(envvars.LD_LIBRARY_PATH, dlp)
+	os.Setenv(envvars.JAVAVM_LD_PRELOAD, "")
+	os.Setenv(envvars.LD_PRELOAD, "")
+	os.Setenv(envvars.MALLOC_ARENA_MAX, "1")
+	util.OptionallyReduceTimerFrequency()
+}
diff --git a/client/go/script-utils/services/prechecks.go b/client/go/script-utils/services/prechecks.go
new file mode 100644
index 00000000000..b41837a8270
--- /dev/null
+++ b/client/go/script-utils/services/prechecks.go
@@ -0,0 +1,37 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"os"
+
+	"github.com/vespa-engine/vespa/client/go/envvars"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+func commonPreChecks() (veHome, veHost string) {
+	if doTrace := os.Getenv(envvars.TRACE_STARTUP); doTrace != "" {
+		trace.AdjustVerbosity(1)
+	}
+	if doDebug := os.Getenv(envvars.DEBUG_STARTUP); doDebug != "" {
+		trace.AdjustVerbosity(2)
+	}
+	_ = vespa.FindAndVerifyVespaHome()
+	err := vespa.LoadDefaultEnv()
+	if err != nil {
+		panic(err)
+	}
+	veHome = vespa.FindAndVerifyVespaHome()
+	veHost, err = vespa.FindOurHostname()
+	if err != nil {
+		trace.Warning("could not detect hostname:", err, "; using fallback:", veHost)
+	}
+	err = os.Chdir(veHome)
+	if err != nil {
+		util.JustExitWith(err)
+	}
+	return
+}
diff --git a/client/go/script-utils/services/sentinel.go b/client/go/script-utils/services/sentinel.go
new file mode 100644
index 00000000000..17f07ce4d55
--- /dev/null
+++ b/client/go/script-utils/services/sentinel.go
@@ -0,0 +1,84 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/vespa-engine/vespa/client/go/envvars"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+const (
+	SENTINEL_PIDFILE      = "var/run/sentinel.pid"
+	SENTINEL_SERVICE_NAME = "config-sentinel"
+)
+
+func startSentinelWithRunserver() {
+	_, veHost := commonPreChecks()
+	vespa.CheckCorrectUser()
+	os.Setenv(envvars.VESPA_SERVICE_NAME, SENTINEL_SERVICE_NAME)
+	configId := fmt.Sprintf("hosts/%s", veHost)
+	args := []string{
+		"-r", "10",
+		"-s", SENTINEL_SERVICE_NAME,
+		"-p", SENTINEL_PIDFILE,
+		"--",
+		"sbin/vespa-config-sentinel",
+		"-c", configId,
+	}
+	cmd := exec.Command("vespa-runserver", args...)
+	cmd.Stdin = nil
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Start()
+	p := cmd.Process
+	if p != nil {
+		p.Release()
+	}
+	os.Unsetenv(envvars.VESPA_SERVICE_NAME)
+}
+
+func waitForSentinelPid() bool {
+	backtick := util.BackTicksWithStderr
+	start := time.Now()
+	for sleepcount := 0; sleepcount < 1000; sleepcount++ {
+		time.Sleep(10 * time.Millisecond)
+		got, err := os.ReadFile(CONFIGPROXY_PIDFILE)
+		if err == nil {
+			pid, err := strconv.Atoi(strings.TrimSpace(string(got)))
+			if err == nil && pid > 0 {
+				_, err := backtick.Run("kill", "-0", strconv.Itoa(pid))
+				if err == nil {
+					secs := int(time.Since(start).Seconds())
+					if secs > 0 {
+						fmt.Printf("config sentinel started after %d seconds\n", secs)
+					}
+					return true
+				}
+			}
+		}
+		if sleepcount%500 == 90 {
+			trace.Warning("Waiting for sentinel to start")
+		}
+	}
+	return false
+}
+
+func StartConfigSentinel() int {
+	commonPreChecks()
+	vespa.MaybeSwitchUser("start-config-sentinel")
+	startSentinelWithRunserver()
+	if waitForSentinelPid() {
+		return 0
+	}
+	return 1
+}
diff --git a/client/go/script-utils/services/start.go b/client/go/script-utils/services/start.go
new file mode 100644
index 00000000000..78375498005
--- /dev/null
+++ b/client/go/script-utils/services/start.go
@@ -0,0 +1,69 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/vespa-engine/vespa/client/go/envvars"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+func StartServices() int {
+	veHome, _ := commonPreChecks()
+	vespa.CheckCorrectUser()
+	trace.Debug("running as correct user")
+	exportSettings(veHome)
+	if vespa.HasOnlyIpV6() {
+		os.Setenv(envvars.VESPA_ONLY_IP_V6_NETWORKING, "true")
+	}
+	startProxyWithRunserver()
+	if waitForProxyResponse() {
+		startSentinelWithRunserver()
+		if waitForSentinelPid() {
+			return 0
+		}
+	}
+	return 1
+}
+
+func checkjava() {
+	backticks := util.BackTicksWithStderr
+	out, err := backticks.Run("java", "-version")
+	if err != nil {
+		trace.Warning("cannot run 'java -version'")
+		util.JustExitWith(err)
+	}
+	if !strings.Contains(out, "64-Bit Server VM") {
+		util.JustExitWith(fmt.Errorf("java must invoke the 64-bit Java VM, but -version says:\n%s\n", out))
+	}
+}
+
+func runvalidation() {
+	// not implemented
+}
+
+func VespaStartServices() int {
+	home, host := commonPreChecks()
+	trace.Debug("common prechecks ok, running in", home, "on", host)
+	vespa.RunPreStart()
+	trace.Debug("prestart ok")
+	util.TuneResourceLimits()
+	trace.Debug("resource limits ok")
+	checkjava()
+	trace.Debug("java ok")
+	runvalidation()
+	enable_transparent_hugepages_with_background_compaction()
+	disable_vm_zone_reclaim_mode()
+	drop_caches()
+	err := vespa.MaybeSwitchUser("start-services")
+	if err != nil {
+		util.JustExitWith(err)
+	}
+	return StartServices()
+}
diff --git a/client/go/script-utils/services/tuning.go b/client/go/script-utils/services/tuning.go
new file mode 100644
index 00000000000..ed0ff509a3d
--- /dev/null
+++ b/client/go/script-utils/services/tuning.go
@@ -0,0 +1,51 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/vespa-engine/vespa/client/go/trace"
+)
+
+const (
+	DROP_CACHES_CTL                = "/proc/sys/vm/drop_caches"
+	TRANSPARENT_HUGEPAGE_ENABLED   = "/sys/kernel/mm/transparent_hugepage/enabled"
+	TRANSPARENT_HUGEPAGE_DEFRAG    = "/sys/kernel/mm/transparent_hugepage/defrag"
+	TRANSPARENT_HUGEPAGE_KH_DEFRAG = "/sys/kernel/mm/transparent_hugepage/khugepaged/defrag"
+	ZONE_RECLAIM_CTL               = "/proc/sys/vm/zone_reclaim_mode"
+)
+
+func maybeEcho(fileName, toWrite string) bool {
+	f, err := os.OpenFile(fileName, os.O_WRONLY, 0)
+	if err == nil {
+		_, err = fmt.Fprintf(f, "%s\n", toWrite)
+		f.Close()
+		if err == nil {
+			trace.Debug("wrote", toWrite, "to", fileName)
+			return true
+		} else {
+			trace.Warning(err)
+		}
+	}
+	return false
+}
+
+func enable_transparent_hugepages_with_background_compaction() {
+	// Should probably also be done on host.
+	maybeEcho(TRANSPARENT_HUGEPAGE_ENABLED, "always")
+	maybeEcho(TRANSPARENT_HUGEPAGE_DEFRAG, "never")
+	maybeEcho(TRANSPARENT_HUGEPAGE_KH_DEFRAG, "1")
+}
+
+func disable_vm_zone_reclaim_mode() {
+	maybeEcho(ZONE_RECLAIM_CTL, "0")
+}
+
+func drop_caches() {
+	if maybeEcho(DROP_CACHES_CTL, "3") {
+		trace.Debug("dropped caches")
+	}
+}