prepare to switch vespa-start-services script

* make it easier to start non-jdisc JVM
* add hooks to start configproxy java program
* add sentinel startup
* port over more from common-env.sh
* tune some logging levels, use spam tracing
* use TRACE_STARTUP=true to get some tracing

17 files changed, 516 insertions, 19 deletions

diff --git a/client/go/defaults/defaults.go b/client/go/defaults/defaults.go
index 83f07af364e..db03e783d58 100644
--- a/client/go/defaults/defaults.go
+++ b/client/go/defaults/defaults.go
@@ -59,6 +59,15 @@ func VespaHostname() string {
 	return DEFAULT_VESPA_HOST
 }
 
+func VespaLogFile() string {
+	if env := os.Getenv(envvars.VESPA_LOG_TARGET); env != "" {
+		if strings.HasPrefix(env, "file:") {
+			return strings.TrimPrefix(env, "file:")
+		}
+	}
+	return UnderVespaHome("logs/vespa/vespa.log")
+}
+
 // Compute the port number where the Vespa webservice
 // container should be available.
 func VespaContainerWebServicePort() int {
diff --git a/client/go/envvars/env_vars.go b/client/go/envvars/env_vars.go
index dfefa3cb51e..ad58338254c 100644
--- a/client/go/envvars/env_vars.go
+++ b/client/go/envvars/env_vars.go
@@ -11,6 +11,7 @@ const (
 	CONFIGPROXY_RPC_PORT                   = "port_configproxy_rpc"
 	CONFIGSERVER_RPC_PORT                  = "port_configserver_rpc"
 	DEBUG_JVM_STARTUP                      = "DEBUG_JVM_STARTUP"
+	DEBUG_STARTUP                          = "DEBUG_STARTUP"
 	FILE_DESCRIPTOR_LIMIT                  = "file_descriptor_limit"
 	GLIBCXX_FORCE_NEW                      = "GLIBCXX_FORCE_NEW"
 	HUGEPAGES_LIST                         = "HUGEPAGES_LIST"
@@ -30,6 +31,7 @@ const (
 	STD_THREAD_PREVENT_TRY_CATCH           = "STD_THREAD_PREVENT_TRY_CATCH"
 	TERM                                   = "TERM"
 	TRACE_JVM_STARTUP                      = "TRACE_JVM_STARTUP"
+	TRACE_STARTUP                          = "TRACE_STARTUP"
 	VESPA_AFFINITY_CPU_SOCKET              = "VESPA_AFFINITY_CPU_SOCKET"
 	VESPA_ALREADY_SWITCHED_USER_TO         = "VESPA_ALREADY_SWITCHED_USER_TO"
 	VESPA_CLI_API_KEY_FILE                 = "VESPA_CLI_API_KEY_FILE"
@@ -44,6 +46,7 @@ const (
 	VESPA_CLI_ENDPOINTS                    = "VESPA_CLI_ENDPOINTS"
 	VESPA_CLI_HOME                         = "VESPA_CLI_HOME"
 	VESPA_CONFIG_ID                        = "VESPA_CONFIG_ID"
+	VESPA_CONFIGPROXY_JVMARGS              = "VESPA_CONFIGPROXY_JVMARGS"
 	VESPA_CONFIGSERVER_JVMARGS             = "VESPA_CONFIGSERVER_JVMARGS"
 	VESPA_CONFIGSERVER_MULTITENANT         = "VESPA_CONFIGSERVER_MULTITENANT"
 	VESPA_CONFIGSERVERS                    = "VESPA_CONFIGSERVERS"
@@ -61,6 +64,7 @@ const (
 	VESPAMALLOC_LIST                       = "VESPAMALLOCD_LIST"
 	VESPA_MALLOC_MADVISE_LIMIT             = "VESPA_MALLOC_MADVISE_LIMIT"
 	VESPA_NO_NUMACTL                       = "VESPA_NO_NUMACTL"
+	VESPA_ONLY_IP_V6_NETWORKING            = "VESPA_ONLY_IP_V6_NETWORKING"
 	VESPA_PORT_BASE                        = "VESPA_PORT_BASE"
 	VESPA_SERVICE_NAME                     = "VESPA_SERVICE_NAME"
 	VESPA_TIMER_HZ                         = "VESPA_TIMER_HZ"
diff --git a/client/go/jvm/configproxy_jvm.go b/client/go/jvm/configproxy_jvm.go
new file mode 100644
index 00000000000..6381ebf6429
--- /dev/null
+++ b/client/go/jvm/configproxy_jvm.go
@@ -0,0 +1,60 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package jvm
+
+import (
+	"strings"
+
+	"github.com/vespa-engine/vespa/client/go/prog"
+)
+
+const (
+	PROXY_JAR_FILE   = "lib/jars/config-proxy-jar-with-dependencies.jar"
+	PROXY_MAIN_CLASS = "com.yahoo.vespa.config.proxy.ProxyServer"
+	PROXY_PORT_ARG   = "19090"
+)
+
+type ConfigProxyJvm struct {
+	containerBase
+}
+
+func (cpc *ConfigProxyJvm) ArgForMain() string {
+	return PROXY_PORT_ARG
+}
+
+func (cpc *ConfigProxyJvm) ConfigId() string {
+	return ""
+}
+
+func (cpc *ConfigProxyJvm) ConfigureOptions(configsources []string, userargs string) {
+	opts := cpc.jvmOpts
+	opts.jarWithDeps = PROXY_JAR_FILE
+	opts.mainClass = PROXY_MAIN_CLASS
+	opts.AddOption("-XX:+ExitOnOutOfMemoryError")
+	opts.AddOption("-XX:+PreserveFramePointer")
+	opts.AddOption("-XX:CompressedClassSpaceSize=32m")
+	opts.AddOption("-XX:MaxDirectMemorySize=32m")
+	opts.AddOption("-XX:ThreadStackSize=448")
+	opts.AddOption("-XX:MaxJavaStackTraceDepth=1000")
+	opts.AddOption("-XX:-OmitStackTraceInFastThrow")
+	opts.AddOption("-XX:ActiveProcessorCount=2")
+	opts.AddOption("-Dproxyconfigsources=" + strings.Join(configsources, ","))
+	opts.AddOption("-Djava.io.tmpdir=${VESPA_HOME}/var/tmp")
+	if userargs != "" {
+		opts.AddJvmArgsFromString(userargs)
+	}
+	minFallback := MegaBytesOfMemory(32)
+	maxFallback := MegaBytesOfMemory(128)
+	opts.AddDefaultHeapSizeArgs(minFallback, maxFallback)
+}
+
+func (cpc *ConfigProxyJvm) exportExtraEnv(ps *prog.Spec) {
+}
+
+func NewConfigProxyJvm(serviceName string) *ConfigProxyJvm {
+	var cpc ConfigProxyJvm
+	cpc.serviceName = serviceName
+	cpc.jvmOpts = NewOptions(&cpc)
+	return &cpc
+}
diff --git a/client/go/jvm/container.go b/client/go/jvm/container.go
index c755672da86..f52a34a6f6d 100644
--- a/client/go/jvm/container.go
+++ b/client/go/jvm/container.go
@@ -65,7 +65,9 @@ func (cb *containerBase) Exec() {
 	p := prog.NewSpec(argv)
 	p.ConfigureNumaCtl()
 	cb.JvmOptions().exportEnvSettings(p)
-	writeEnvAsProperties(p.EffectiveEnv(), cb.propsFile)
+	if cb.propsFile != "" {
+		writeEnvAsProperties(p.EffectiveEnv(), cb.propsFile)
+	}
 	trace.Info("starting container; env:", readableEnv(p.Env))
 	trace.Info("starting container; exec:", argv)
 	err := p.Run()
diff --git a/client/go/jvm/jdisc_options.go b/client/go/jvm/jdisc_options.go
index 84348bbda63..8d06f5c5a6c 100644
--- a/client/go/jvm/jdisc_options.go
+++ b/client/go/jvm/jdisc_options.go
@@ -8,8 +8,8 @@ import (
 )
 
 const (
-	DEFAULT_MAIN_CLASS = "com.yahoo.jdisc.core.StandaloneMain"
-	DEFAULT_CP_FILE    = "lib/jars/jdisc_core-jar-with-dependencies.jar"
+	DEFAULT_MAIN_CLASS    = "com.yahoo.jdisc.core.StandaloneMain"
+	DEFAULT_JAR_WITH_DEPS = "lib/jars/jdisc_core-jar-with-dependencies.jar"
 )
 
 func (opts *Options) AddCommonJdiscProperties() {
diff --git a/client/go/jvm/jdk_properties.go b/client/go/jvm/jdk_properties.go
index 7656d87db3d..70888bf49c7 100644
--- a/client/go/jvm/jdk_properties.go
+++ b/client/go/jvm/jdk_properties.go
@@ -4,7 +4,10 @@
 package jvm
 
 import (
+	"os"
+
 	"github.com/vespa-engine/vespa/client/go/defaults"
+	"github.com/vespa-engine/vespa/client/go/envvars"
 )
 
 func (opts *Options) AddCommonJdkProperties() {
@@ -23,4 +26,7 @@ func (opts *Options) AddCommonJdkProperties() {
 	opts.AddOption("-Djdk.tls.rejectClientInitiatedRenegotiation=true")
 	opts.AddOption("-Dfile.encoding=UTF-8")
 	opts.AddOption("-Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger")
+	if env := os.Getenv(envvars.VESPA_ONLY_IP_V6_NETWORKING); env == "true" {
+		opts.AddOption("-Djava.net.preferIPv6Addresses=true")
+	}
 }
diff --git a/client/go/jvm/options.go b/client/go/jvm/options.go
index 55749e86709..4ece59caa31 100644
--- a/client/go/jvm/options.go
+++ b/client/go/jvm/options.go
@@ -15,11 +15,12 @@ import (
 )
 
 type Options struct {
-	container Container
-	classPath []string
-	jvmArgs   util.ArrayList[string]
-	mainClass string
-	fixSpec   util.FixSpec
+	container   Container
+	classPath   []string
+	jvmArgs     util.ArrayList[string]
+	mainClass   string
+	jarWithDeps string
+	fixSpec     util.FixSpec
 }
 
 func NewOptions(c Container) *Options {
@@ -31,11 +32,12 @@ func NewOptions(c Container) *Options {
 		FileMode: 0644,
 	}
 	return &Options{
-		container: c,
-		classPath: make([]string, 0, 10),
-		jvmArgs:   make([]string, 0, 100),
-		mainClass: DEFAULT_MAIN_CLASS,
-		fixSpec:   fixSpec,
+		container:   c,
+		classPath:   make([]string, 0, 10),
+		jvmArgs:     make([]string, 0, 100),
+		jarWithDeps: DEFAULT_JAR_WITH_DEPS,
+		mainClass:   DEFAULT_MAIN_CLASS,
+		fixSpec:     fixSpec,
 	}
 }
 
@@ -52,7 +54,7 @@ func (opts *Options) AppendOption(arg string) {
 }
 
 func (opts *Options) ClassPath() string {
-	cp := defaults.UnderVespaHome(DEFAULT_CP_FILE)
+	cp := defaults.UnderVespaHome(opts.jarWithDeps)
 	for _, x := range opts.classPath {
 		cp = fmt.Sprintf("%s:%s", cp, x)
 	}
diff --git a/client/go/script-utils/main.go b/client/go/script-utils/main.go
index 97f31957b36..89802ad5697 100644
--- a/client/go/script-utils/main.go
+++ b/client/go/script-utils/main.go
@@ -13,6 +13,7 @@ import (
 	"github.com/vespa-engine/vespa/client/go/cmd/logfmt"
 	"github.com/vespa-engine/vespa/client/go/jvm"
 	"github.com/vespa-engine/vespa/client/go/script-utils/configserver"
+	"github.com/vespa-engine/vespa/client/go/script-utils/services"
 	"github.com/vespa-engine/vespa/client/go/script-utils/standalone"
 	"github.com/vespa-engine/vespa/client/go/script-utils/startcbinary"
 	"github.com/vespa-engine/vespa/client/go/util"
@@ -33,6 +34,12 @@ func main() {
 		os.Args = os.Args[1:]
 	}
 	switch action {
+	case "vespa-start-services":
+		os.Exit(services.VespaStartServices())
+	case "start-services":
+		os.Exit(services.StartServices())
+	case "just-run-configproxy":
+		os.Exit(services.JustRunConfigproxy())
 	case "vespa-start-configserver":
 		os.Exit(configserver.StartConfigserverEtc())
 	case "just-start-configserver":
diff --git a/client/go/script-utils/services/configproxy.go b/client/go/script-utils/services/configproxy.go
new file mode 100644
index 00000000000..fb452a4de00
--- /dev/null
+++ b/client/go/script-utils/services/configproxy.go
@@ -0,0 +1,134 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/vespa-engine/vespa/client/go/defaults"
+	"github.com/vespa-engine/vespa/client/go/envvars"
+	"github.com/vespa-engine/vespa/client/go/jvm"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+const (
+	CONFIGPROXY_PIDFILE = "var/run/configproxy.pid"
+	PROXY_SERVICE_NAME  = "configproxy"
+)
+
+func JustRunConfigproxy() int {
+	commonPreChecks()
+	vespa.CheckCorrectUser()
+	configsources := defaults.VespaConfigserverRpcAddrs()
+	if len(configsources) < 1 {
+		util.JustExitMsg("could not find any configservers")
+	}
+	util.TuneResourceLimits()
+	c := jvm.NewConfigProxyJvm(PROXY_SERVICE_NAME)
+	userargs := os.Getenv(envvars.VESPA_CONFIGPROXY_JVMARGS)
+	c.ConfigureOptions(configsources, userargs)
+	c.Exec()
+	// unreachable:
+	return 1
+}
+
+func startProxyWithRunserver() {
+	commonPreChecks()
+	vespa.CheckCorrectUser()
+	configsources := defaults.VespaConfigserverRpcAddrs()
+	fmt.Printf(
+		"Starting config proxy using %s as config source(s)\n",
+		strings.Join(configsources, " and "))
+	args := []string{
+		"-r", "10",
+		"-s", PROXY_SERVICE_NAME,
+		"-p", CONFIGPROXY_PIDFILE,
+		"--",
+		"libexec/vespa/script-utils", "just-run-configproxy",
+	}
+	cmd := exec.Command("vespa-runserver", args...)
+	cmd.Stdin = nil
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Start()
+	p := cmd.Process
+	if p != nil {
+		p.Release()
+	}
+}
+
+func waitForProxyResponse() bool {
+	hname, _ := vespa.FindOurHostname()
+	backtick := util.BackTicksWithStderr
+	start := time.Now()
+	fmt.Printf("Waiting for config proxy to start\n")
+	for sleepcount := 0; sleepcount < 1800; sleepcount++ {
+		time.Sleep(100 * time.Millisecond)
+		got, err := os.ReadFile(CONFIGPROXY_PIDFILE)
+		if err == nil {
+			pid, err := strconv.Atoi(strings.TrimSpace(string(got)))
+			if err == nil && pid > 0 {
+				out, err := backtick.Run("vespa-ping-configproxy", "-s", hname)
+				if err == nil {
+					secs := time.Since(start).Seconds()
+					fmt.Printf("config proxy started after %ds (runserver pid %d)\n", int(secs), pid)
+					return true
+				}
+				if sleepcount%50 == 9 {
+					trace.Warning("Could not ping configproxy:", err)
+					if sleepcount%500 == 59 {
+						secs := time.Since(start).Seconds()
+						trace.Warning("ping output after", int(secs), "seconds:", strings.TrimSpace(out))
+						logFile := defaults.VespaLogFile()
+						cmd := fmt.Sprintf("tail -n 15 %s | vespa-logfmt -l all -N", logFile)
+						out, err = backtick.Run("sh", "-c", cmd)
+						fmt.Fprintf(os.Stderr, "tail of logfile: >>>\n%s<<<\n", out)
+					}
+				}
+			} else {
+				trace.Debug("bad contents (", string(got), ") in pid file", CONFIGPROXY_PIDFILE)
+			}
+		} else {
+			trace.Debug("bad pid file", CONFIGPROXY_PIDFILE, err)
+		}
+	}
+	secs := time.Since(start).Seconds()
+	fmt.Fprintf(os.Stderr, "Config proxy still failed to start after %d seconds!\n", int(secs))
+	got, err := os.ReadFile(CONFIGPROXY_PIDFILE)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "pid file %s was not created\n", CONFIGPROXY_PIDFILE)
+		return false
+	}
+	gotpid := strings.TrimSpace(string(got))
+	pid, err := strconv.Atoi(gotpid)
+	if err != nil || pid < 1 {
+		fmt.Fprintf(os.Stderr, "invalid pid '%s' in file %s\n", gotpid, CONFIGPROXY_PIDFILE)
+		return false
+	}
+	out, err := backtick.Run("kill", "-0", gotpid)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "config proxy process `%s` has terminated: %s\n", gotpid, strings.TrimSpace(out))
+		return false
+	}
+	out, err = backtick.Run("vespa-ping-configproxy", "-s", hname)
+	fmt.Fprintf(os.Stderr, "failed to ping configproxy: %s\n", out)
+	return false
+}
+
+func StartConfigproxy() int {
+	commonPreChecks()
+	vespa.MaybeSwitchUser("start-configproxy")
+	startProxyWithRunserver()
+	if waitForProxyResponse() {
+		return 0
+	}
+	return 1
+}
diff --git a/client/go/script-utils/services/env.go b/client/go/script-utils/services/env.go
new file mode 100644
index 00000000000..1a51d1a21c7
--- /dev/null
+++ b/client/go/script-utils/services/env.go
@@ -0,0 +1,25 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/vespa-engine/vespa/client/go/envvars"
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+func exportSettings(vespaHome string) {
+	vlt := fmt.Sprintf("file:%s/logs/vespa/vespa.log", vespaHome)
+	lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome)
+	dlp := fmt.Sprintf("%s/lib64:/opt/vespa-deps/lib64", vespaHome)
+	os.Setenv(envvars.VESPA_LOG_TARGET, vlt)
+	os.Setenv(envvars.VESPA_LOG_CONTROL_DIR, lcd)
+	os.Setenv(envvars.LD_LIBRARY_PATH, dlp)
+	os.Setenv(envvars.JAVAVM_LD_PRELOAD, "")
+	os.Setenv(envvars.LD_PRELOAD, "")
+	os.Setenv(envvars.MALLOC_ARENA_MAX, "1")
+	util.OptionallyReduceTimerFrequency()
+}
diff --git a/client/go/script-utils/services/prechecks.go b/client/go/script-utils/services/prechecks.go
new file mode 100644
index 00000000000..b41837a8270
--- /dev/null
+++ b/client/go/script-utils/services/prechecks.go
@@ -0,0 +1,37 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"os"
+
+	"github.com/vespa-engine/vespa/client/go/envvars"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+func commonPreChecks() (veHome, veHost string) {
+	if doTrace := os.Getenv(envvars.TRACE_STARTUP); doTrace != "" {
+		trace.AdjustVerbosity(1)
+	}
+	if doDebug := os.Getenv(envvars.DEBUG_STARTUP); doDebug != "" {
+		trace.AdjustVerbosity(2)
+	}
+	_ = vespa.FindAndVerifyVespaHome()
+	err := vespa.LoadDefaultEnv()
+	if err != nil {
+		panic(err)
+	}
+	veHome = vespa.FindAndVerifyVespaHome()
+	veHost, err = vespa.FindOurHostname()
+	if err != nil {
+		trace.Warning("could not detect hostname:", err, "; using fallback:", veHost)
+	}
+	err = os.Chdir(veHome)
+	if err != nil {
+		util.JustExitWith(err)
+	}
+	return
+}
diff --git a/client/go/script-utils/services/sentinel.go b/client/go/script-utils/services/sentinel.go
new file mode 100644
index 00000000000..17f07ce4d55
--- /dev/null
+++ b/client/go/script-utils/services/sentinel.go
@@ -0,0 +1,84 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/vespa-engine/vespa/client/go/envvars"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+const (
+	SENTINEL_PIDFILE      = "var/run/sentinel.pid"
+	SENTINEL_SERVICE_NAME = "config-sentinel"
+)
+
+func startSentinelWithRunserver() {
+	_, veHost := commonPreChecks()
+	vespa.CheckCorrectUser()
+	os.Setenv(envvars.VESPA_SERVICE_NAME, SENTINEL_SERVICE_NAME)
+	configId := fmt.Sprintf("hosts/%s", veHost)
+	args := []string{
+		"-r", "10",
+		"-s", SENTINEL_SERVICE_NAME,
+		"-p", SENTINEL_PIDFILE,
+		"--",
+		"sbin/vespa-config-sentinel",
+		"-c", configId,
+	}
+	cmd := exec.Command("vespa-runserver", args...)
+	cmd.Stdin = nil
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Start()
+	p := cmd.Process
+	if p != nil {
+		p.Release()
+	}
+	os.Unsetenv(envvars.VESPA_SERVICE_NAME)
+}
+
+func waitForSentinelPid() bool {
+	backtick := util.BackTicksWithStderr
+	start := time.Now()
+	for sleepcount := 0; sleepcount < 1000; sleepcount++ {
+		time.Sleep(10 * time.Millisecond)
+		got, err := os.ReadFile(CONFIGPROXY_PIDFILE)
+		if err == nil {
+			pid, err := strconv.Atoi(strings.TrimSpace(string(got)))
+			if err == nil && pid > 0 {
+				_, err := backtick.Run("kill", "-0", strconv.Itoa(pid))
+				if err == nil {
+					secs := int(time.Since(start).Seconds())
+					if secs > 0 {
+						fmt.Printf("config sentinel started after %d seconds\n", secs)
+					}
+					return true
+				}
+			}
+		}
+		if sleepcount%500 == 90 {
+			trace.Warning("Waiting for sentinel to start")
+		}
+	}
+	return false
+}
+
+func StartConfigSentinel() int {
+	commonPreChecks()
+	vespa.MaybeSwitchUser("start-config-sentinel")
+	startSentinelWithRunserver()
+	if waitForSentinelPid() {
+		return 0
+	}
+	return 1
+}
diff --git a/client/go/script-utils/services/start.go b/client/go/script-utils/services/start.go
new file mode 100644
index 00000000000..78375498005
--- /dev/null
+++ b/client/go/script-utils/services/start.go
@@ -0,0 +1,69 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/vespa-engine/vespa/client/go/envvars"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+func StartServices() int {
+	veHome, _ := commonPreChecks()
+	vespa.CheckCorrectUser()
+	trace.Debug("running as correct user")
+	exportSettings(veHome)
+	if vespa.HasOnlyIpV6() {
+		os.Setenv(envvars.VESPA_ONLY_IP_V6_NETWORKING, "true")
+	}
+	startProxyWithRunserver()
+	if waitForProxyResponse() {
+		startSentinelWithRunserver()
+		if waitForSentinelPid() {
+			return 0
+		}
+	}
+	return 1
+}
+
+func checkjava() {
+	backticks := util.BackTicksWithStderr
+	out, err := backticks.Run("java", "-version")
+	if err != nil {
+		trace.Warning("cannot run 'java -version'")
+		util.JustExitWith(err)
+	}
+	if !strings.Contains(out, "64-Bit Server VM") {
+		util.JustExitWith(fmt.Errorf("java must invoke the 64-bit Java VM, but -version says:\n%s\n", out))
+	}
+}
+
+func runvalidation() {
+	// not implemented
+}
+
+func VespaStartServices() int {
+	home, host := commonPreChecks()
+	trace.Debug("common prechecks ok, running in", home, "on", host)
+	vespa.RunPreStart()
+	trace.Debug("prestart ok")
+	util.TuneResourceLimits()
+	trace.Debug("resource limits ok")
+	checkjava()
+	trace.Debug("java ok")
+	runvalidation()
+	enable_transparent_hugepages_with_background_compaction()
+	disable_vm_zone_reclaim_mode()
+	drop_caches()
+	err := vespa.MaybeSwitchUser("start-services")
+	if err != nil {
+		util.JustExitWith(err)
+	}
+	return StartServices()
+}
diff --git a/client/go/script-utils/services/tuning.go b/client/go/script-utils/services/tuning.go
new file mode 100644
index 00000000000..ed0ff509a3d
--- /dev/null
+++ b/client/go/script-utils/services/tuning.go
@@ -0,0 +1,51 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package services
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/vespa-engine/vespa/client/go/trace"
+)
+
+const (
+	DROP_CACHES_CTL                = "/proc/sys/vm/drop_caches"
+	TRANSPARENT_HUGEPAGE_ENABLED   = "/sys/kernel/mm/transparent_hugepage/enabled"
+	TRANSPARENT_HUGEPAGE_DEFRAG    = "/sys/kernel/mm/transparent_hugepage/defrag"
+	TRANSPARENT_HUGEPAGE_KH_DEFRAG = "/sys/kernel/mm/transparent_hugepage/khugepaged/defrag"
+	ZONE_RECLAIM_CTL               = "/proc/sys/vm/zone_reclaim_mode"
+)
+
+func maybeEcho(fileName, toWrite string) bool {
+	f, err := os.OpenFile(fileName, os.O_WRONLY, 0)
+	if err == nil {
+		_, err = fmt.Fprintf(f, "%s\n", toWrite)
+		f.Close()
+		if err == nil {
+			trace.Debug("wrote", toWrite, "to", fileName)
+			return true
+		} else {
+			trace.Warning(err)
+		}
+	}
+	return false
+}
+
+func enable_transparent_hugepages_with_background_compaction() {
+	// Should probably also be done on host.
+	maybeEcho(TRANSPARENT_HUGEPAGE_ENABLED, "always")
+	maybeEcho(TRANSPARENT_HUGEPAGE_DEFRAG, "never")
+	maybeEcho(TRANSPARENT_HUGEPAGE_KH_DEFRAG, "1")
+}
+
+func disable_vm_zone_reclaim_mode() {
+	maybeEcho(ZONE_RECLAIM_CTL, "0")
+}
+
+func drop_caches() {
+	if maybeEcho(DROP_CACHES_CTL, "3") {
+		trace.Debug("dropped caches")
+	}
+}
diff --git a/client/go/trace/log.go b/client/go/trace/log.go
index 3a8fa1a4846..b89b1f82322 100644
--- a/client/go/trace/log.go
+++ b/client/go/trace/log.go
@@ -21,6 +21,9 @@ func logMessage(l outputLevel, msg string) {
 	hostname := os.Getenv(envvars.VESPA_HOSTNAME)
 	pid := os.Getpid()
 	service := os.Getenv(envvars.VESPA_SERVICE_NAME)
+	if service == "" {
+		service = "-"
+	}
 	component := "stderr"
 	level := "error"
 	switch l {
diff --git a/client/go/trace/trace.go b/client/go/trace/trace.go
index 31eeeca841c..bf180e80dad 100644
--- a/client/go/trace/trace.go
+++ b/client/go/trace/trace.go
@@ -50,6 +50,10 @@ func Debug(v ...interface{}) {
 	outputTracing(levelDebug, v...)
 }
 
+func SpamDebug(v ...interface{}) {
+	outputTracing(levelSpam, v...)
+}
+
 func Warning(v ...interface{}) {
 	outputTracing(levelWarning, v...)
 }
diff --git a/client/go/util/fix_fs.go b/client/go/util/fix_fs.go
index fcbcefd0d71..57caa7f1158 100644
--- a/client/go/util/fix_fs.go
+++ b/client/go/util/fix_fs.go
@@ -48,7 +48,7 @@ func statNoSymlinks(path string) (info os.FileInfo, err error) {
 		if (info.Mode() & os.ModeSymlink) != 0 {
 			return nil, fmt.Errorf("the path '%s' is a symlink, not allowed", name)
 		}
-		trace.Debug("lstat", name, "=>", info.Mode(), err)
+		trace.SpamDebug("lstat", name, "=>", info.Mode(), err)
 	}
 	return info, err
 }
@@ -71,13 +71,13 @@ func (spec *FixSpec) FixDir(dirName string) {
 		err = fmt.Errorf("Not a directory: '%s'", dirName)
 		spec.complainAndExit(err, dirName, spec.DirMode)
 	}
-	trace.Debug("chown: ", dirName, spec.UserId, spec.GroupId)
+	trace.SpamDebug("chown: ", dirName, spec.UserId, spec.GroupId)
 	err = os.Chown(dirName, spec.UserId, spec.GroupId)
 	if err != nil {
 		spec.ensureWritableDir(dirName)
 		return
 	}
-	trace.Debug("chmod: ", dirName, spec.DirMode)
+	trace.SpamDebug("chmod: ", dirName, spec.DirMode)
 	err = os.Chmod(dirName, spec.DirMode)
 	if err != nil {
 		spec.ensureWritableDir(dirName)
@@ -98,13 +98,13 @@ func (spec *FixSpec) FixFile(fileName string) {
 		err = fmt.Errorf("Should not be a directory: '%s'", fileName)
 		spec.complainAndExit(err, fileName, spec.FileMode)
 	}
-	trace.Debug("chown: ", fileName, spec.UserId, spec.GroupId)
+	trace.SpamDebug("chown: ", fileName, spec.UserId, spec.GroupId)
 	err = os.Chown(fileName, spec.UserId, spec.GroupId)
 	if err != nil {
 		spec.ensureWritableFile(fileName)
 		return
 	}
-	trace.Debug("chmod: ", fileName, spec.FileMode)
+	trace.SpamDebug("chmod: ", fileName, spec.FileMode)
 	err = os.Chmod(fileName, spec.FileMode)
 	if err != nil {
 		spec.ensureWritableFile(fileName)