add configserver startup

10 files changed, 394 insertions, 3 deletions

diff --git a/client/go/script-utils/configserver/check.go b/client/go/script-utils/configserver/check.go
new file mode 100644
index 00000000000..450a81104a4
--- /dev/null
+++ b/client/go/script-utils/configserver/check.go
@@ -0,0 +1,24 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package configserver
+
+import (
+	"fmt"
+
+	"github.com/vespa-engine/vespa/client/go/defaults"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+func checkIsConfigserver(myname string) {
+	onlyHosts := defaults.VespaConfigserverHosts()
+	for _, hn := range onlyHosts {
+		if hn == "localhost" || hn == myname {
+			trace.Debug("should run configserver:", hn)
+			return
+		}
+	}
+	trace.Warning("only these hosts should run a config server:", onlyHosts)
+	util.JustExitMsg(fmt.Sprintf("this host [%s] should not run a config server", myname))
+}
diff --git a/client/go/script-utils/configserver/env.go b/client/go/script-utils/configserver/env.go
new file mode 100644
index 00000000000..ce4dd8c3c64
--- /dev/null
+++ b/client/go/script-utils/configserver/env.go
@@ -0,0 +1,30 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package configserver
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+func exportSettings(vespaHome string) {
+	vlt := fmt.Sprintf("file:%s/logs/vespa/vespa.log", vespaHome)
+	lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome)
+	lcf := fmt.Sprintf("%s/configserver.logcontrol", lcd)
+	dlp := fmt.Sprintf("%s/lib64", vespaHome)
+	app := fmt.Sprintf("%s/conf/configserver-app", vespaHome)
+	os.Setenv("VESPA_LOG_TARGET", vlt)
+	os.Setenv("VESPA_LOG_CONTROL_DIR", lcd)
+	os.Setenv("VESPA_LOG_CONTROL_FILE", lcf)
+	os.Setenv("VESPA_SERVICE_NAME", "configserver")
+	os.Setenv("LD_LIBRARY_PATH", dlp)
+	os.Setenv("JAVAVM_LD_PRELOAD", "")
+	os.Setenv("LD_PRELOAD", "")
+	os.Setenv("standalone_jdisc_container__app_location", app)
+	os.Setenv("standalone_jdisc_container__deployment_profile", "configserver")
+	os.Setenv("MALLOC_ARENA_MAX", "1")
+	util.OptionallyReduceTimerFrequency()
+}
diff --git a/client/go/script-utils/configserver/fix_dirs_and_files.go b/client/go/script-utils/configserver/fix_dirs_and_files.go
new file mode 100644
index 00000000000..73c4c73334e
--- /dev/null
+++ b/client/go/script-utils/configserver/fix_dirs_and_files.go
@@ -0,0 +1,27 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package configserver
+
+import (
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+func makeFixSpec() util.FixSpec {
+	vespaUid, vespaGid := vespa.FindVespaUidAndGid()
+	return util.FixSpec{
+		UserId:   vespaUid,
+		GroupId:  vespaGid,
+		DirMode:  0755,
+		FileMode: 0644,
+	}
+}
+
+func fixDirsAndFiles(fixSpec util.FixSpec) {
+	fixSpec.FixDir("conf/zookeeper")
+	fixSpec.FixDir("var/zookeeper")
+	fixSpec.FixDir("var/zookeeper/version-2")
+	fixSpec.FixFile("conf/zookeeper/zookeeper.cfg")
+	fixSpec.FixFile("var/zookeeper/myid")
+}
diff --git a/client/go/script-utils/configserver/logd.go b/client/go/script-utils/configserver/logd.go
new file mode 100644
index 00000000000..7eed70b23da
--- /dev/null
+++ b/client/go/script-utils/configserver/logd.go
@@ -0,0 +1,24 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package configserver
+
+import (
+	"os"
+
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+func maybeStartLogd() {
+	v1 := os.Getenv("cloudconfig_server__multitenant")
+	v2 := os.Getenv("VESPA_CONFIGSERVER_MULTITENANT")
+	if v1 == "true" || v2 == "true" {
+		backticks := util.BackTicksForwardStderr
+		out, err := backticks.Run("libexec/vespa/start-logd")
+		if err != nil {
+			panic(err)
+		}
+		trace.Info(out)
+	}
+}
diff --git a/client/go/script-utils/configserver/runserver.go b/client/go/script-utils/configserver/runserver.go
new file mode 100644
index 00000000000..bf498941128
--- /dev/null
+++ b/client/go/script-utils/configserver/runserver.go
@@ -0,0 +1,60 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package configserver
+
+import (
+	"fmt"
+
+	"github.com/vespa-engine/vespa/client/go/defaults"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+const (
+	PROG_NAME = "vespa-runserver"
+)
+
+type RunServer struct {
+	ServiceName string
+	Args        []string
+}
+
+func (rs *RunServer) PidFile() string {
+	varRunDir := defaults.UnderVespaHome("var/run")
+	return fmt.Sprintf("%s/%s.pid", varRunDir, rs.ServiceName)
+}
+
+func (rs *RunServer) ProgPath() string {
+	p := fmt.Sprintf("%s/bin64/%s", defaults.VespaHome(), PROG_NAME)
+	if util.IsExecutableFile(p) {
+		return p
+	}
+	p = fmt.Sprintf("%s/bin/%s", defaults.VespaHome(), PROG_NAME)
+	if util.IsExecutableFile(p) {
+		return p
+	}
+	panic(fmt.Errorf("not an executable file: %s", p))
+}
+
+func (rs *RunServer) WouldRun() bool {
+	backticks := util.BackTicksForwardStderr
+	out, err := backticks.Run(rs.ProgPath(), "-s", rs.ServiceName, "-p", rs.PidFile(), "-W")
+	trace.Trace("output from -W:", out, "error:", err)
+	return err == nil
+}
+
+func (rs *RunServer) Exec(prog string) {
+	argv := []string{
+		PROG_NAME,
+		"-s", rs.ServiceName,
+		"-p", rs.PidFile(),
+		"--",
+		prog,
+	}
+	for _, arg := range rs.Args {
+		argv = append(argv, arg)
+	}
+	err := util.Execvp(rs.ProgPath(), argv)
+	util.JustExitWith(err)
+}
diff --git a/client/go/script-utils/configserver/start.go b/client/go/script-utils/configserver/start.go
new file mode 100644
index 00000000000..cff38471d53
--- /dev/null
+++ b/client/go/script-utils/configserver/start.go
@@ -0,0 +1,84 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package configserver
+
+import (
+	"os"
+
+	"github.com/vespa-engine/vespa/client/go/jvm"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+const (
+	SERVICE_NAME = "configserver"
+)
+
+func commonPreChecks() (veHome string) {
+	if doTrace := os.Getenv("TRACE_JVM_STARTUP"); doTrace != "" {
+		trace.AdjustVerbosity(1)
+	}
+	if doDebug := os.Getenv("DEBUG_JVM_STARTUP"); doDebug != "" {
+		trace.AdjustVerbosity(2)
+	}
+	_ = vespa.FindAndVerifyVespaHome()
+	err := vespa.LoadDefaultEnv()
+	if err != nil {
+		panic(err)
+	}
+	veHome = vespa.FindAndVerifyVespaHome()
+	veHost, e := vespa.FindOurHostname()
+	if e != nil {
+		trace.Warning("could not detect hostname:", err, "; using fallback:", veHost)
+	}
+	checkIsConfigserver(veHost)
+	e = os.Chdir(veHome)
+	if e != nil {
+		util.JustExitWith(e)
+	}
+	return
+}
+
+func JustStartConfigserver() int {
+	vespaHome := commonPreChecks()
+	vespa.CheckCorrectUser()
+	util.TuneResourceLimits()
+	exportSettings(vespaHome)
+	removeStaleZkLocks(vespaHome)
+	c := jvm.NewStandaloneContainer(SERVICE_NAME)
+	jvmOpts := c.JvmOptions()
+	if extra := os.Getenv("VESPA_CONFIGSERVER_JVMARGS"); extra != "" {
+		jvmOpts.AddJvmArgsFromString(extra)
+	}
+	minFallback := jvm.MegaBytesOfMemory(128)
+	maxFallback := jvm.MegaBytesOfMemory(2048)
+	jvmOpts.AddDefaultHeapSizeArgs(minFallback, maxFallback)
+	c.Exec()
+	// unreachable:
+	return 1
+}
+
+func runConfigserverWithRunserver() int {
+	commonPreChecks()
+	vespa.CheckCorrectUser()
+	rs := RunServer{
+		ServiceName: SERVICE_NAME,
+		Args:        []string{"just-start-configserver"},
+	}
+	rs.Exec("libexec/vespa/script-utils")
+	return 1
+}
+
+func StartConfigserverEtc() int {
+	vespaHome := commonPreChecks()
+	vespa.RunPreStart()
+	util.TuneResourceLimits()
+	fixSpec := makeFixSpec()
+	fixDirsAndFiles(fixSpec)
+	exportSettings(vespaHome)
+	vespa.MaybeSwitchUser("vespa-start-configserver")
+	maybeStartLogd()
+	return runConfigserverWithRunserver()
+}
diff --git a/client/go/script-utils/configserver/zk.go b/client/go/script-utils/configserver/zk.go
new file mode 100644
index 00000000000..f8a8514364d
--- /dev/null
+++ b/client/go/script-utils/configserver/zk.go
@@ -0,0 +1,22 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package configserver
+
+import (
+	"fmt"
+
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+const (
+	ZOOKEEPER_LOG_FILE_PREFIX = "logs/vespa/zookeeper.configserver"
+)
+
+func removeStaleZkLocks(vespaHome string) {
+	backticks := util.BackTicksIgnoreStderr
+	cmd := fmt.Sprintf("rm -f '%s/%s'*lck", vespaHome, ZOOKEEPER_LOG_FILE_PREFIX)
+	trace.Trace("cleaning locks:", cmd)
+	backticks.Run("/bin/sh", "-c", cmd)
+}
diff --git a/client/go/script-utils/main.go b/client/go/script-utils/main.go
index 99f2d00067b..713a208a319 100644
--- a/client/go/script-utils/main.go
+++ b/client/go/script-utils/main.go
@@ -12,6 +12,7 @@ import (
 	"github.com/vespa-engine/vespa/client/go/cmd/deploy"
 	"github.com/vespa-engine/vespa/client/go/cmd/logfmt"
 	"github.com/vespa-engine/vespa/client/go/jvm"
+	"github.com/vespa-engine/vespa/client/go/script-utils/configserver"
 	"github.com/vespa-engine/vespa/client/go/script-utils/startcbinary"
 	"github.com/vespa-engine/vespa/client/go/util"
 	"github.com/vespa-engine/vespa/client/go/vespa"
@@ -31,6 +32,10 @@ func main() {
 		os.Args = os.Args[1:]
 	}
 	switch action {
+	case "vespa-start-configserver":
+		os.Exit(configserver.StartConfigserverEtc())
+	case "just-start-configserver":
+		os.Exit(configserver.JustStartConfigserver())
 	case "vespa-start-container-daemon":
 		os.Exit(jvm.RunApplicationContainer(os.Args[1:]))
 	case "start-c-binary":
diff --git a/client/go/vespa/prestart.go b/client/go/vespa/prestart.go
new file mode 100644
index 00000000000..eb4a598ef37
--- /dev/null
+++ b/client/go/vespa/prestart.go
@@ -0,0 +1,72 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package vespa
+
+import (
+	"io/fs"
+	"os"
+
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+func RunPreStart() error {
+	vespaHome := FindAndVerifyVespaHome()
+	err := LoadDefaultEnv()
+	if err != nil {
+		return err
+	}
+	trace.Trace("chdir:", vespaHome)
+	err = os.Chdir(vespaHome)
+	if err != nil {
+		return err
+	}
+	vespaUid, vespaGid := FindVespaUidAndGid()
+	fixSpec := util.FixSpec{
+		UserId:   vespaUid,
+		GroupId:  vespaGid,
+		DirMode:  0755,
+		FileMode: 0644,
+	}
+	fixSpec.FixDir("logs")
+	fixSpec.FixDir("logs/vespa")
+	fixSpec.FixDir("logs/vespa/access")
+	fixSpec.FixDir("logs/vespa/configserver")
+	fixSpec.FixDir("logs/vespa/search")
+	fixSpec.FixDir("var/tmp")
+	fixSpec.FixDir("var/tmp/vespa")
+	fixSpec.FixDir("var")
+	fixSpec.FixDir("var/crash")
+	fixSpec.FixDir("var/db/vespa")
+	fixSpec.FixDir("var/db/vespa/config_server")
+	fixSpec.FixDir("var/db/vespa/config_server/serverdb")
+	fixSpec.FixDir("var/db/vespa/config_server/serverdb/tenants")
+	fixSpec.FixDir("var/db/vespa/filedistribution")
+	fixSpec.FixDir("var/db/vespa/index")
+	fixSpec.FixDir("var/db/vespa/logcontrol")
+	fixSpec.FixDir("var/db/vespa/search")
+	fixSpec.FixDir("var/db/vespa/tmp")
+	fixSpec.FixDir("var/jdisc_container")
+	fixSpec.FixDir("var/run")
+	fixSpec.FixDir("var/vespa")
+	fixSpec.FixDir("var/vespa/application")
+	fixSpec.FixDir("var/vespa/bundlecache")
+	fixSpec.FixDir("var/vespa/bundlecache/configserver")
+	fixSpec.FixDir("var/vespa/cache/config")
+	var fixer fs.WalkDirFunc = func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			panic(err)
+		}
+		if d.IsDir() {
+			fixSpec.FixDir(path)
+		} else if d.Type().IsRegular() {
+			fixSpec.FixFile(path)
+		}
+		return nil
+	}
+	fileSystem := os.DirFS(vespaHome)
+	fs.WalkDir(fileSystem, "logs/vespa", fixer)
+	fs.WalkDir(fileSystem, "var/db/vespa", fixer)
+	return nil
+}
diff --git a/client/go/vespa/switch_user.go b/client/go/vespa/switch_user.go
index 20d624965f8..79677e9c145 100644
--- a/client/go/vespa/switch_user.go
+++ b/client/go/vespa/switch_user.go
@@ -13,11 +13,54 @@ import (
 	"github.com/vespa-engine/vespa/client/go/util"
 )
 
+const ENV_CHECK = "VESPA_ALREADY_SWITCHED_USER_TO"
+
+func CheckCorrectUser() {
+	vespaUser := FindVespaUser()
+	currentName := ""
+	currentUser, err1 := user.Current()
+	if err1 == nil {
+		currentName = currentUser.Username
+	} else {
+		trace.Trace("user.Current() failed:", err1)
+	}
+	if currentName == vespaUser {
+		// all OK
+		return
+	}
+	trace.Trace("FindVespaUser():", vespaUser, "!=", currentName)
+	wantName := ""
+	wantUser, err2 := user.Lookup(vespaUser)
+	if err2 == nil {
+		wantName = wantUser.Username
+	} else {
+		trace.Trace("user.Lookup", vespaUser, "failed:", err2)
+	}
+	if currentName == wantName {
+		// somewhat OK
+		return
+	}
+	trace.Warning("not running as the VESPA_USER:", vespaUser)
+	alreadyTried := os.Getenv(ENV_CHECK)
+	if alreadyTried != "" {
+		trace.Warning("already tried to switch user to", alreadyTried)
+	}
+	if wantName != vespaUser {
+		trace.Warning("alternate correct user:", wantName)
+	}
+	if err1 != nil {
+		trace.Warning("note: user.Current() failed:", err1)
+	}
+	if err2 != nil {
+		trace.Warning("note: user.Lookup(", vespaUser, ") failed:", err2)
+	}
+	util.JustExitMsg("running as wrong user. Check your VESPA_USER setting")
+}
+
 // re-execute a script-utils action after switching to the vespa user
 // (used by vespa-start-configserver and vespa-start-services)
 func MaybeSwitchUser(action string) error {
 	const SU_PROG = "vespa-run-as-vespa-user"
-	const ENV_CHECK = "VESPA_ALREADY_SWITCHED_USER_TO"
 	vespaHome := FindHome()
 	vespaUser := FindVespaUser()
 
@@ -31,9 +74,9 @@ func MaybeSwitchUser(action string) error {
 		trace.Trace("user.Current() failed:", err)
 		return err
 	}
-	trace.Trace("want to switch user from:", currUser.Username)
-	trace.Trace("want to switch user to:", wantUser.Username)
 	if wantUser.Username != currUser.Username {
+		trace.Trace("want to switch user from:", currUser.Username)
+		trace.Trace("want to switch user to:", wantUser.Username)
 		alreadyTried := os.Getenv(ENV_CHECK)
 		if alreadyTried != "" {
 			// safety check to avoid infinite loop