more generic prog.Spec

17 files changed, 782 insertions, 0 deletions

diff --git a/client/go/prog/common_env.go b/client/go/prog/common_env.go
new file mode 100644
index 00000000000..f8c8ad54def
--- /dev/null
+++ b/client/go/prog/common_env.go
@@ -0,0 +1,79 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package prog
+
+import (
+	"os"
+	"strings"
+
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+const (
+	ENV_JAVA_HOME        = util.ENV_JAVA_HOME
+	ENV_LD_LIBRARY_PATH  = util.ENV_LD_LIBRARY_PATH
+	ENV_LD_PRELOAD       = util.ENV_LD_PRELOAD
+	ENV_MALLOC_ARENA_MAX = util.ENV_MALLOC_ARENA_MAX
+	ENV_PATH             = util.ENV_PATH
+	ENV_ROOT             = util.ENV_ROOT
+	ENV_VESPA_USER       = util.ENV_VESPA_USER
+
+	ENV_VESPA_AFFINITY_CPU_SOCKET    = "VESPA_AFFINITY_CPU_SOCKET"
+	ENV_VESPA_LOAD_CODE_AS_HUGEPAGES = "VESPA_LOAD_CODE_AS_HUGEPAGES"
+	ENV_VESPA_MALLOC_HUGEPAGES       = "VESPA_MALLOC_HUGEPAGES"
+	ENV_VESPA_MALLOC_MADVISE_LIMIT   = "VESPA_MALLOC_MADVISE_LIMIT"
+	ENV_VESPA_NO_NUMACTL             = "VESPA_NO_NUMACTL"
+	ENV_VESPA_TIMER_HZ               = "VESPA_TIMER_HZ"
+	ENV_VESPA_USE_HUGEPAGES          = "VESPA_USE_HUGEPAGES"
+	ENV_VESPA_USE_HUGEPAGES_LIST     = "VESPA_USE_HUGEPAGES_LIST"
+	ENV_VESPA_USE_MADVISE_LIST       = "VESPA_USE_MADVISE_LIST"
+	ENV_VESPA_USE_NO_VESPAMALLOC     = "VESPA_USE_NO_VESPAMALLOC"
+	ENV_VESPA_USE_VALGRIND           = "VESPA_USE_VALGRIND"
+	ENV_VESPA_USE_VESPAMALLOC        = "VESPA_USE_VESPAMALLOC"
+	ENV_VESPA_USE_VESPAMALLOC_D      = "VESPA_USE_VESPAMALLOC_D"
+	ENV_VESPA_USE_VESPAMALLOC_DST    = "VESPA_USE_VESPAMALLOC_DST"
+	ENV_VESPA_VALGRIND_OPT           = "VESPA_VALGRIND_OPT"
+
+	// backwards compatibility variables:
+	ENV_GLIBCXX_FORCE_NEW            = "GLIBCXX_FORCE_NEW"
+	ENV_HUGEPAGES_LIST               = "HUGEPAGES_LIST"
+	ENV_MADVISE_LIST                 = "MADVISE_LIST"
+	ENV_NO_VESPAMALLOC_LIST          = "NO_VESPAMALLOC_LIST"
+	ENV_STD_THREAD_PREVENT_TRY_CATCH = "STD_THREAD_PREVENT_TRY_CATCH"
+	ENV_VESPAMALLOCDST_LIST          = "VESPAMALLOCDST_LIST"
+	ENV_VESPAMALLOCD_LIST            = "VESPAMALLOCD_LIST"
+	ENV_VESPAMALLOC_LIST             = "VESPAMALLOCD_LIST"
+)
+
+func (spec *Spec) configureCommonEnv() {
+	os.Unsetenv(ENV_LD_PRELOAD)
+	spec.Setenv(ENV_STD_THREAD_PREVENT_TRY_CATCH, "true")
+	spec.Setenv(ENV_GLIBCXX_FORCE_NEW, "1")
+	spec.Setenv(ENV_LD_LIBRARY_PATH, vespa.FindHome()+"/lib64")
+	spec.Setenv(ENV_MALLOC_ARENA_MAX, "1")
+
+	// fallback from old env.vars:
+	spec.considerEnvFallback(ENV_VESPA_USE_HUGEPAGES_LIST, ENV_HUGEPAGES_LIST)
+	spec.considerEnvFallback(ENV_VESPA_USE_MADVISE_LIST, ENV_MADVISE_LIST)
+	spec.considerEnvFallback(ENV_VESPA_USE_VESPAMALLOC, ENV_VESPAMALLOC_LIST)
+	spec.considerEnvFallback(ENV_VESPA_USE_VESPAMALLOC_D, ENV_VESPAMALLOCD_LIST)
+	spec.considerEnvFallback(ENV_VESPA_USE_VESPAMALLOC_DST, ENV_VESPAMALLOCDST_LIST)
+	spec.considerEnvFallback(ENV_VESPA_USE_NO_VESPAMALLOC, ENV_NO_VESPAMALLOC_LIST)
+	// other fallbacks:
+	spec.considerFallback(ENV_ROOT, vespa.FindHome())
+	spec.considerFallback(ENV_VESPA_USER, vespa.FindVespaUser())
+	spec.considerFallback(ENV_VESPA_USE_HUGEPAGES_LIST, "all")
+	spec.considerFallback(ENV_VESPA_USE_VESPAMALLOC, "all")
+	spec.considerFallback(ENV_VESPA_USE_NO_VESPAMALLOC, strings.Join([]string{
+		"vespa-rpc-invoke",
+		"vespa-get-config",
+		"vespa-sentinel-cmd",
+		"vespa-route",
+		"vespa-proton-cmd",
+		"vespa-configproxy-cmd",
+		"vespa-config-status",
+	}, " "))
+
+}
diff --git a/client/go/prog/hugepages.go b/client/go/prog/hugepages.go
new file mode 100644
index 00000000000..79c5f4d7854
--- /dev/null
+++ b/client/go/prog/hugepages.go
@@ -0,0 +1,15 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package prog
+
+import (
+	"github.com/vespa-engine/vespa/client/go/trace"
+)
+
+func (spec *Spec) ConfigureHugePages() {
+	if spec.matchesListEnv(ENV_VESPA_USE_HUGEPAGES_LIST) {
+		trace.Debug("setting", ENV_VESPA_USE_HUGEPAGES, "= 'yes'")
+		spec.Setenv(ENV_VESPA_USE_HUGEPAGES, "yes")
+	}
+}
diff --git a/client/go/prog/madvise.go b/client/go/prog/madvise.go
new file mode 100644
index 00000000000..fda6aa3bea0
--- /dev/null
+++ b/client/go/prog/madvise.go
@@ -0,0 +1,17 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package prog
+
+import (
+	"github.com/vespa-engine/vespa/client/go/trace"
+)
+
+func (spec *Spec) ConfigureUseMadvise() {
+	limit := spec.valueFromListEnv(ENV_VESPA_USE_MADVISE_LIST)
+	if limit != "" {
+		trace.Trace("shall use madvise with limit", limit, "as set in", ENV_VESPA_USE_MADVISE_LIST)
+		spec.Setenv(ENV_VESPA_MALLOC_MADVISE_LIMIT, limit)
+		return
+	}
+}
diff --git a/client/go/prog/mockbin/bad-numactl b/client/go/prog/mockbin/bad-numactl
new file mode 100755
index 00000000000..1b08bb19b95
--- /dev/null
+++ b/client/go/prog/mockbin/bad-numactl
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# emulate numactl which says hardware is available but fails anyway
+
+case $#.$* in
+    1.--hardware)
+        cat << 'EOF'
+available: 2 nodes (0-1)
+node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46
+node 0 size: 128604 MB
+node 0 free: 107167 MB
+node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39 41 43 45 47
+node 1 size: 128976 MB
+node 1 free: 110921 MB
+node distances:
+node   0   1 
+  0:  10  21 
+  1:  21  10 
+EOF
+        ;;
+    *)
+        echo "numactl failure"
+        ;;
+esac
+exit 0
diff --git a/client/go/prog/mockbin/good-numactl b/client/go/prog/mockbin/good-numactl
new file mode 100755
index 00000000000..f861809b382
--- /dev/null
+++ b/client/go/prog/mockbin/good-numactl
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# emulate normal numactl operations
+
+case $#.$* in
+    1.--hardware)
+        cat << 'EOF'
+available: 2 nodes (0-1)
+node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46
+node 0 size: 128604 MB
+node 0 free: 107167 MB
+node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39 41 43 45 47
+node 1 size: 128976 MB
+node 1 free: 110921 MB
+node distances:
+node   0   1 
+  0:  10  21 
+  1:  21  10 
+EOF
+        ;;
+    *)
+        echo "foo"
+        ;;
+esac
+exit 0
diff --git a/client/go/prog/mockbin/has-valgrind b/client/go/prog/mockbin/has-valgrind
new file mode 100755
index 00000000000..83601cd64d1
--- /dev/null
+++ b/client/go/prog/mockbin/has-valgrind
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# emulate working valgrind
+
+echo "some help text here"
+exit 0
diff --git a/client/go/prog/mockbin/no-numactl b/client/go/prog/mockbin/no-numactl
new file mode 100755
index 00000000000..dfe583abaca
--- /dev/null
+++ b/client/go/prog/mockbin/no-numactl
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# emulate inoperational numactl
+
+echo "No NUMA available on this system"
+exit 1
diff --git a/client/go/prog/mockbin/no-valgrind b/client/go/prog/mockbin/no-valgrind
new file mode 100755
index 00000000000..09a2e517c37
--- /dev/null
+++ b/client/go/prog/mockbin/no-valgrind
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# emulate bad or missing valgrind
+
+echo "some error message here"
+exit 1
diff --git a/client/go/prog/numactl.go b/client/go/prog/numactl.go
new file mode 100644
index 00000000000..6fda925908c
--- /dev/null
+++ b/client/go/prog/numactl.go
@@ -0,0 +1,73 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package prog
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+const (
+	NUMACTL_PROG = "numactl"
+)
+
+func (p *Spec) ConfigureNumaCtl() {
+	p.shouldUseNumaCtl = false
+	p.numaSocket = -1
+	if p.Getenv(ENV_VESPA_NO_NUMACTL) != "" {
+		return
+	}
+	backticks := util.BackTicksIgnoreStderr
+	out, err := backticks.Run(NUMACTL_PROG, "--hardware")
+	trace.Debug("numactl --hardware says:", out)
+	if err != nil {
+		trace.Trace("numactl error:", err)
+		return
+	}
+	outfoo, errfoo := backticks.Run(NUMACTL_PROG, "--interleave", "all", "echo", "foo")
+	if errfoo != nil {
+		trace.Trace("cannot run with numactl:", errfoo)
+		return
+	}
+	if outfoo != "foo\n" {
+		trace.Trace("bad numactl output:", outfoo)
+		return
+	}
+	p.shouldUseNumaCtl = true
+	if affinity := p.Getenv(ENV_VESPA_AFFINITY_CPU_SOCKET); affinity != "" {
+		wantSocket, _ := strconv.Atoi(affinity)
+		trace.Debug("want socket:", wantSocket)
+		parts := strings.Fields(out)
+		for idx := 0; idx+2 < len(parts); idx++ {
+			if parts[idx] == "available:" && parts[idx+2] == "nodes" {
+				numSockets, _ := strconv.Atoi(parts[idx+1])
+				trace.Debug("numSockets:", numSockets)
+				if numSockets > 1 {
+					p.numaSocket = (wantSocket % numSockets)
+					return
+				}
+			}
+		}
+	}
+}
+
+func (p *Spec) prependNumaCtl(args []string) []string {
+	result := make([]string, 0, 5+len(args))
+	result = append(result, NUMACTL_PROG)
+	if p.numaSocket >= 0 {
+		result = append(result, fmt.Sprintf("--cpunodebind=%d", p.numaSocket))
+		result = append(result, fmt.Sprintf("--membind=%d", p.numaSocket))
+	} else {
+		result = append(result, "--interleave")
+		result = append(result, "all")
+	}
+	for _, arg := range args {
+		result = append(result, arg)
+	}
+	return result
+}
diff --git a/client/go/prog/numactl_test.go b/client/go/prog/numactl_test.go
new file mode 100644
index 00000000000..b6f6a7d8506
--- /dev/null
+++ b/client/go/prog/numactl_test.go
@@ -0,0 +1,90 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package prog
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vespa-engine/vespa/client/go/trace"
+)
+
+func setup(t *testing.T, testFileName string) {
+	trace.AdjustVerbosity(1)
+	mockBinParent = strings.TrimSuffix(testFileName, "/numactl_test.go")
+	tmpBin = t.TempDir() + "/mock.bin.numactl_test"
+	err := os.MkdirAll(tmpBin, 0755)
+	assert.Nil(t, err)
+	t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH")))
+}
+
+func TestNumaCtlDetection(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		return
+	}
+	_, tfn, _, _ := runtime.Caller(0)
+	setup(t, tfn)
+	orig := []string{"/bin/myprog", "-c", "cfgid"}
+	spec := NewSpec(orig)
+
+	useMock("no-numactl", "numactl")
+	spec.ConfigureNumaCtl()
+	assert.Equal(t, false, spec.shouldUseNumaCtl)
+
+	useMock("bad-numactl", "numactl")
+	spec.ConfigureNumaCtl()
+	assert.Equal(t, false, spec.shouldUseNumaCtl)
+
+	t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "")
+	useMock("good-numactl", "numactl")
+	spec.ConfigureNumaCtl()
+	assert.Equal(t, true, spec.shouldUseNumaCtl)
+	assert.Equal(t, -1, spec.numaSocket)
+	argv := spec.prependNumaCtl(orig)
+	trace.Trace("argv:", argv)
+	assert.Equal(t, 6, len(argv))
+	assert.Equal(t, "numactl", argv[0])
+	assert.Equal(t, "--interleave", argv[1])
+	assert.Equal(t, "all", argv[2])
+	assert.Equal(t, "/bin/myprog", argv[3])
+	assert.Equal(t, "-c", argv[4])
+	assert.Equal(t, "cfgid", argv[5])
+
+	t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "0")
+	spec.ConfigureNumaCtl()
+	assert.Equal(t, true, spec.shouldUseNumaCtl)
+	assert.Equal(t, 0, spec.numaSocket)
+	argv = spec.prependNumaCtl(orig)
+	trace.Trace("argv:", argv)
+	assert.Equal(t, 6, len(argv))
+	assert.Equal(t, "numactl", argv[0])
+	assert.Equal(t, "--cpunodebind=0", argv[1])
+	assert.Equal(t, "--membind=0", argv[2])
+	assert.Equal(t, "/bin/myprog", argv[3])
+	assert.Equal(t, "-c", argv[4])
+	assert.Equal(t, "cfgid", argv[5])
+
+	t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "1")
+	spec.ConfigureNumaCtl()
+	assert.Equal(t, true, spec.shouldUseNumaCtl)
+	assert.Equal(t, 1, spec.numaSocket)
+	argv = spec.prependNumaCtl(orig)
+	trace.Trace("argv:", argv)
+	assert.Equal(t, 6, len(argv))
+	assert.Equal(t, "numactl", argv[0])
+	assert.Equal(t, "--cpunodebind=1", argv[1])
+	assert.Equal(t, "--membind=1", argv[2])
+	assert.Equal(t, "/bin/myprog", argv[3])
+	assert.Equal(t, "-c", argv[4])
+	assert.Equal(t, "cfgid", argv[5])
+
+	t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "2")
+	spec.ConfigureNumaCtl()
+	assert.Equal(t, true, spec.shouldUseNumaCtl)
+	assert.Equal(t, 0, spec.numaSocket)
+
+}
diff --git a/client/go/prog/run.go b/client/go/prog/run.go
new file mode 100644
index 00000000000..49d1b626cb4
--- /dev/null
+++ b/client/go/prog/run.go
@@ -0,0 +1,25 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package prog
+
+import (
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+func (spec *Spec) Run() error {
+	prog := spec.Program
+	args := spec.Args
+	if spec.shouldUseValgrind {
+		args = spec.prependValgrind(args)
+		prog = args[0]
+	} else if spec.shouldUseNumaCtl {
+		args = spec.prependNumaCtl(args)
+		prog = args[0]
+	}
+	if spec.shouldUseVespaMalloc {
+		spec.Setenv(ENV_LD_PRELOAD, spec.vespaMallocPreload)
+	}
+	envv := spec.effectiveEnv()
+	return util.Execvpe(prog, args, envv)
+}
diff --git a/client/go/prog/spec.go b/client/go/prog/spec.go
new file mode 100644
index 00000000000..cf553470a44
--- /dev/null
+++ b/client/go/prog/spec.go
@@ -0,0 +1,39 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package prog
+
+import (
+	"strings"
+)
+
+type Spec struct {
+	Program              string
+	Args                 []string
+	BaseName             string
+	Env                  map[string]string
+	numaSocket           int
+	shouldUseCallgrind   bool
+	shouldUseValgrind    bool
+	shouldUseNumaCtl     bool
+	shouldUseVespaMalloc bool
+	vespaMallocPreload   string
+}
+
+func NewSpec(argv []string) *Spec {
+	progName := argv[0]
+	p := Spec{
+		Program:    progName,
+		Args:       argv,
+		BaseName:   baseNameOf(progName),
+		Env:        make(map[string]string),
+		numaSocket: -1,
+	}
+	return &p
+}
+
+func baseNameOf(s string) string {
+	idx := strings.LastIndex(s, "/")
+	idx++
+	return s[idx:]
+}
diff --git a/client/go/prog/spec_env.go b/client/go/prog/spec_env.go
new file mode 100644
index 00000000000..498931fe92c
--- /dev/null
+++ b/client/go/prog/spec_env.go
@@ -0,0 +1,104 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package prog
+
+import (
+	"os"
+	"strings"
+
+	"github.com/vespa-engine/vespa/client/go/trace"
+)
+
+func (p *Spec) Setenv(k, v string) {
+	p.Env[k] = v
+}
+
+func (p *Spec) Getenv(k string) string {
+	if v, ok := p.Env[k]; ok {
+		return v
+	}
+	return os.Getenv(k)
+}
+
+func (spec *Spec) effectiveEnv() []string {
+	env := make(map[string]string)
+	for _, entry := range os.Environ() {
+		addInMap := func(kv string) bool {
+			for idx, elem := range kv {
+				if elem == '=' {
+					k := kv[:idx]
+					env[k] = kv
+					return true
+				}
+			}
+			return false
+		}
+		if !addInMap(entry) {
+			env[entry] = ""
+		}
+	}
+	for k, v := range spec.Env {
+		trace.Trace("add to environment:", k, "=", v)
+		env[k] = k + "=" + v
+	}
+	envv := make([]string, 0, len(env))
+	for _, v := range env {
+		envv = append(envv, v)
+	}
+	return envv
+}
+
+func (spec *Spec) considerFallback(varName, varValue string) {
+	if spec.Getenv(varName) == "" && varValue != "" {
+		spec.Setenv(varName, varValue)
+	}
+}
+
+func (spec *Spec) considerEnvFallback(targetVar, fallbackVar string) {
+	spec.considerFallback(targetVar, spec.Getenv(fallbackVar))
+}
+
+func (p *Spec) matchesListEnv(envVarName string) bool {
+	return p.matchesListString(p.Getenv(envVarName))
+}
+
+func (p *Spec) matchesListString(env string) bool {
+	if env == "all" {
+		trace.Debug(p.Program, "always matching in:", env)
+		return true
+	}
+	parts := strings.Fields(env)
+	for _, part := range parts {
+		if p.BaseName == part {
+			trace.Debug(p.Program, "has basename matching in:", env)
+			return true
+		}
+		trace.Debug("checking matching:", p.BaseName, "!=", part)
+	}
+	return false
+}
+
+func (p *Spec) valueFromListEnv(envVarName string) string {
+	return p.valueFromListString(p.Getenv(envVarName))
+}
+
+func (p *Spec) valueFromListString(env string) string {
+	parts := strings.Fields(env)
+	for _, part := range parts {
+		idx := strings.Index(part, "=")
+		if idx <= 0 {
+			trace.Trace("expected key=value, but got:", part)
+			continue
+		}
+		partName := part[:idx]
+		idx++
+		partValue := part[idx:]
+		if p.BaseName == partName || partName == "all" {
+			trace.Debug(p.Program, "has basename matching in:", env)
+			return partValue
+		}
+		trace.Debug("checking matching:", p.BaseName, "!=", part)
+	}
+	return ""
+}
diff --git a/client/go/prog/spec_test.go b/client/go/prog/spec_test.go
new file mode 100644
index 00000000000..a4002051a86
--- /dev/null
+++ b/client/go/prog/spec_test.go
@@ -0,0 +1,39 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package prog
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProgSpec(t *testing.T) {
+	spec := NewSpec([]string{"/opt/vespa/bin/foobar"})
+	var b bool
+
+	b = spec.matchesListString("")
+	assert.Equal(t, false, b)
+	b = spec.matchesListString("foobar")
+	assert.Equal(t, true, b)
+	b = spec.matchesListString("foo bar")
+	assert.Equal(t, false, b)
+	b = spec.matchesListString("one foobar")
+	assert.Equal(t, true, b)
+	b = spec.matchesListString("foobar two")
+	assert.Equal(t, true, b)
+	b = spec.matchesListString("one foobar two")
+	assert.Equal(t, true, b)
+	b = spec.matchesListString("all")
+	assert.Equal(t, true, b)
+
+	var s string
+	s = spec.valueFromListString("")
+	assert.Equal(t, "", s)
+	s = spec.valueFromListString("foobar=123")
+	assert.Equal(t, "123", s)
+	s = spec.valueFromListString("one=1 foobar=123 two=2")
+	assert.Equal(t, "123", s)
+	s = spec.valueFromListString("one=1 all=123")
+	assert.Equal(t, "123", s)
+}
diff --git a/client/go/prog/valgrind.go b/client/go/prog/valgrind.go
new file mode 100644
index 00000000000..da893fed1f5
--- /dev/null
+++ b/client/go/prog/valgrind.go
@@ -0,0 +1,85 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package prog
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+const (
+	VALGRIND_PROG = "valgrind"
+)
+
+func (p *Spec) ConfigureValgrind() {
+	p.shouldUseValgrind = false
+	p.shouldUseCallgrind = false
+	env := p.Getenv(ENV_VESPA_USE_VALGRIND)
+	parts := strings.Split(env, " ")
+	for _, part := range parts {
+		if p.BaseName == part {
+			trace.Trace("using valgrind as", p.Program, "has basename in", ENV_VESPA_USE_VALGRIND, "=>", env)
+			backticks := util.BackTicksWithStderr
+			out, err := backticks.Run(VALGRIND_PROG, "--help")
+			if err != nil {
+				trace.Trace("trial run of valgrind fails:", err, "=>", out)
+				return
+			}
+			if opts := p.Getenv(ENV_VESPA_VALGRIND_OPT); strings.Contains(opts, "callgrind") {
+				p.shouldUseCallgrind = true
+			}
+			p.shouldUseValgrind = true
+			return
+		}
+		trace.Debug("checking", ENV_VESPA_USE_VALGRIND, ":", p.BaseName, "!=", part)
+	}
+}
+
+func (p *Spec) valgrindOptions() []string {
+	env := p.Getenv(ENV_VESPA_VALGRIND_OPT)
+	if env != "" {
+		return strings.Fields(env)
+	}
+	result := []string{
+		"--num-callers=32",
+		"--run-libc-freeres=yes",
+		"--track-origins=yes",
+		"--freelist-vol=1000000000",
+		"--leak-check=full",
+		"--show-reachable=yes",
+	}
+	result = addValgrindSuppression(result, "etc/vespa/valgrind-suppressions.txt")
+	result = addValgrindSuppression(result, "etc/vespa/suppressions.txt")
+	return result
+}
+
+func addValgrindSuppression(r []string, fn string) []string {
+	existsOk, fileName := vespa.HasFileUnderVespaHome(fn)
+	if existsOk {
+		r = append(r, fmt.Sprintf("--suppressions=%s", fileName))
+	}
+	return r
+}
+
+func (p *Spec) valgrindLogOption() string {
+	return fmt.Sprintf("--log-file=%s/var/tmp/valgrind.%s.log.%d", vespa.FindHome(), p.BaseName, os.Getpid())
+}
+
+func (p *Spec) prependValgrind(args []string) []string {
+	result := make([]string, 0, 15+len(args))
+	result = append(result, VALGRIND_PROG)
+	for _, arg := range p.valgrindOptions() {
+		result = append(result, arg)
+	}
+	result = append(result, p.valgrindLogOption())
+	for _, arg := range args {
+		result = append(result, arg)
+	}
+	return result
+}
diff --git a/client/go/prog/valgrind_test.go b/client/go/prog/valgrind_test.go
new file mode 100644
index 00000000000..786efcebd0c
--- /dev/null
+++ b/client/go/prog/valgrind_test.go
@@ -0,0 +1,92 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package prog
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/util"
+)
+
+var tmpBin string
+var mockBinParent string
+
+func useMock(prog, target string) {
+	mock := fmt.Sprintf("%s/mockbin/%s", mockBinParent, prog)
+	symlink := fmt.Sprintf("%s/%s", tmpBin, target)
+	os.Remove(symlink)
+	err := os.Symlink(mock, symlink)
+	if err != nil {
+		util.JustExitWith(err)
+	}
+}
+
+func setupValgrind(t *testing.T, testFileName string) {
+	trace.AdjustVerbosity(1)
+	t.Setenv("VESPA_HOME", mockBinParent+"/mock_vespahome")
+	mockBinParent = strings.TrimSuffix(testFileName, "/valgrind_test.go")
+	tmpBin = t.TempDir() + "/mock.bin.valgrind_test"
+	err := os.MkdirAll(tmpBin, 0755)
+	assert.Nil(t, err)
+	t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH")))
+}
+
+func TestValgrindDetection(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		return
+	}
+	_, tfn, _, _ := runtime.Caller(0)
+	setupValgrind(t, tfn)
+	spec := NewSpec([]string{"/opt/vespa/bin/foobar"})
+	var argv []string
+
+	useMock("has-valgrind", VALGRIND_PROG)
+
+	t.Setenv("VESPA_USE_VALGRIND", "")
+	spec.ConfigureValgrind()
+	assert.Equal(t, false, spec.shouldUseValgrind)
+	assert.Equal(t, false, spec.shouldUseCallgrind)
+
+	t.Setenv("VESPA_USE_VALGRIND", "foo bar")
+	spec.ConfigureValgrind()
+	assert.Equal(t, false, spec.shouldUseValgrind)
+	assert.Equal(t, false, spec.shouldUseCallgrind)
+
+	t.Setenv("VESPA_USE_VALGRIND", "foobar")
+	spec.ConfigureValgrind()
+	assert.Equal(t, true, spec.shouldUseValgrind)
+	assert.Equal(t, false, spec.shouldUseCallgrind)
+
+	argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"})
+	trace.Trace("argv:", argv)
+	assert.Equal(t, 11, len(argv))
+	assert.Equal(t, "valgrind", argv[0])
+	assert.Equal(t, "/bin/myprog", argv[8])
+
+	t.Setenv("VESPA_USE_VALGRIND", "another foobar yetmore")
+	spec.ConfigureValgrind()
+	assert.Equal(t, true, spec.shouldUseValgrind)
+	assert.Equal(t, false, spec.shouldUseCallgrind)
+
+	t.Setenv("VESPA_VALGRIND_OPT", "--tool=callgrind")
+	spec.ConfigureValgrind()
+	assert.Equal(t, true, spec.shouldUseValgrind)
+	assert.Equal(t, true, spec.shouldUseCallgrind)
+
+	argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"})
+	trace.Trace("argv:", argv)
+	assert.Equal(t, 6, len(argv))
+	assert.Equal(t, "valgrind", argv[0])
+	assert.Equal(t, "--tool=callgrind", argv[1])
+	assert.Equal(t, "/bin/myprog", argv[3])
+
+	useMock("no-valgrind", "valgrind")
+	spec.ConfigureValgrind()
+	assert.Equal(t, false, spec.shouldUseValgrind)
+	assert.Equal(t, false, spec.shouldUseCallgrind)
+}
diff --git a/client/go/prog/vespamalloc.go b/client/go/prog/vespamalloc.go
new file mode 100644
index 00000000000..94952548507
--- /dev/null
+++ b/client/go/prog/vespamalloc.go
@@ -0,0 +1,56 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package prog
+
+import (
+	"fmt"
+
+	"github.com/vespa-engine/vespa/client/go/trace"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+)
+
+func vespaMallocLib(suf string) string {
+	prefixes := []string{"lib64", "lib"}
+	for _, pre := range prefixes {
+		fn := fmt.Sprintf("%s/vespa/malloc/%s", pre, suf)
+		existsOk, fileName := vespa.HasFileUnderVespaHome(fn)
+		if existsOk {
+			trace.Debug("found library:", fileName)
+			return fileName
+		}
+		trace.Debug("bad or missing library:", fn)
+	}
+	return ""
+}
+
+func (p *Spec) ConfigureVespaMalloc() {
+	p.shouldUseVespaMalloc = false
+	if p.matchesListEnv(ENV_VESPA_USE_NO_VESPAMALLOC) {
+		trace.Trace("use no vespamalloc:", p.BaseName)
+		return
+	}
+	if p.shouldUseValgrind && !p.shouldUseCallgrind {
+		trace.Trace("use valgrind, so no vespamalloc:", p.BaseName)
+		return
+	}
+	var useFile string
+	if p.matchesListEnv(ENV_VESPA_USE_VESPAMALLOC_DST) {
+		useFile = vespaMallocLib("libvespamallocdst16.so")
+	} else if p.matchesListEnv(ENV_VESPA_USE_VESPAMALLOC_D) {
+		useFile = vespaMallocLib("libvespamallocd.so")
+	} else if p.matchesListEnv(ENV_VESPA_USE_VESPAMALLOC) {
+		useFile = vespaMallocLib("libvespamalloc.so")
+	}
+	trace.Trace("use file:", useFile)
+	if useFile == "" {
+		return
+	}
+	if loadAsHuge := p.Getenv(ENV_VESPA_LOAD_CODE_AS_HUGEPAGES); loadAsHuge != "" {
+		otherFile := vespaMallocLib("libvespa_load_as_huge.so")
+		useFile = fmt.Sprintf("%s:%s", useFile, otherFile)
+	}
+	p.considerEnvFallback(ENV_VESPA_MALLOC_HUGEPAGES, ENV_VESPA_USE_HUGEPAGES)
+	p.vespaMallocPreload = useFile
+	p.shouldUseVespaMalloc = true
+}