Rename util -> osutil

10 files changed, 620 insertions, 0 deletions

diff --git a/client/go/internal/osutil/execvp.go b/client/go/internal/osutil/execvp.go
new file mode 100644
index 00000000000..331b8166428
--- /dev/null
+++ b/client/go/internal/osutil/execvp.go
@@ -0,0 +1,47 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+//go:build !windows
+
+package osutil
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/vespa-engine/vespa/client/go/internal/admin/envvars"
+	"github.com/vespa-engine/vespa/client/go/internal/admin/trace"
+	"github.com/vespa-engine/vespa/client/go/internal/ioutil"
+	"golang.org/x/sys/unix"
+)
+
+func findInPath(prog string) string {
+	if strings.Contains(prog, "/") {
+		return prog
+	}
+	path := strings.Split(os.Getenv(envvars.PATH), ":")
+	for _, dir := range path {
+		fn := dir + "/" + prog
+		if ioutil.IsExecutable(fn) {
+			return fn
+		}
+	}
+	return prog
+}
+
+func Execvp(prog string, argv []string) error {
+	return Execvpe(prog, argv, os.Environ())
+}
+
+func Execvpe(prog string, argv []string, envv []string) error {
+	prog = findInPath(prog)
+	argv[0] = prog
+	return Execve(prog, argv, envv)
+}
+
+func Execve(prog string, argv []string, envv []string) error {
+	trace.Trace("run cmd:", strings.Join(argv, " "))
+	err := unix.Exec(prog, argv, envv)
+	return fmt.Errorf("cannot execute '%s': %v", prog, err)
+}
diff --git a/client/go/internal/osutil/execvp_windows.go b/client/go/internal/osutil/execvp_windows.go
new file mode 100644
index 00000000000..0e8e7a4a673
--- /dev/null
+++ b/client/go/internal/osutil/execvp_windows.go
@@ -0,0 +1,22 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+//go:build windows
+
+package osutil
+
+import (
+	"fmt"
+)
+
+func Execvp(prog string, argv []string) error {
+	return fmt.Errorf("cannot execvp on windows: %s", prog)
+}
+
+func Execvpe(prog string, argv []string, envv []string) error {
+	return fmt.Errorf("cannot execvp on windows: %s", prog)
+}
+
+func Execve(prog string, argv []string, envv []string) error {
+	return fmt.Errorf("cannot execvp on windows: %s", prog)
+}
diff --git a/client/go/internal/osutil/fix_fs.go b/client/go/internal/osutil/fix_fs.go
new file mode 100644
index 00000000000..837624cc05b
--- /dev/null
+++ b/client/go/internal/osutil/fix_fs.go
@@ -0,0 +1,153 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package osutil
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/user"
+	"strings"
+
+	"github.com/vespa-engine/vespa/client/go/internal/admin/trace"
+)
+
+type FixSpec struct {
+	UserId   int
+	GroupId  int
+	DirMode  os.FileMode
+	FileMode os.FileMode
+}
+
+func NewFixSpec() FixSpec {
+	return FixSpec{
+		UserId:   -1,
+		GroupId:  -1,
+		DirMode:  0755,
+		FileMode: 0644,
+	}
+}
+
+func statNoSymlinks(path string) (info os.FileInfo, err error) {
+	components := strings.Split(path, "/")
+	var name string
+	for idx, x := range components {
+		if idx == 0 {
+			name = x
+			if x == "" {
+				continue
+			}
+		} else {
+			name = name + "/" + x
+		}
+		info, err = os.Lstat(name)
+		if err != nil {
+			return
+		}
+		if (info.Mode() & os.ModeSymlink) != 0 {
+			return nil, fmt.Errorf("the path '%s' is a symlink, not allowed", name)
+		}
+		trace.SpamDebug("lstat", name, "=>", info.Mode(), err)
+	}
+	return info, err
+}
+
+// ensure directory exists with suitable permissions
+func (spec *FixSpec) FixDir(dirName string) {
+	info, err := statNoSymlinks(dirName)
+	if errors.Is(err, os.ErrNotExist) {
+		trace.Trace("mkdir: ", dirName)
+		err = os.MkdirAll(dirName, spec.DirMode)
+		if err != nil {
+			spec.complainAndExit(err, dirName, spec.DirMode)
+		}
+		info, err = statNoSymlinks(dirName)
+	}
+	if err != nil {
+		spec.complainAndExit(err, dirName, spec.DirMode)
+	}
+	if !info.IsDir() {
+		err = fmt.Errorf("Not a directory: '%s'", dirName)
+		spec.complainAndExit(err, dirName, spec.DirMode)
+	}
+	trace.SpamDebug("chown: ", dirName, spec.UserId, spec.GroupId)
+	err = os.Chown(dirName, spec.UserId, spec.GroupId)
+	if err != nil {
+		spec.ensureWritableDir(dirName)
+		return
+	}
+	trace.SpamDebug("chmod: ", dirName, spec.DirMode)
+	err = os.Chmod(dirName, spec.DirMode)
+	if err != nil {
+		spec.ensureWritableDir(dirName)
+	}
+	trace.Debug("directory ok:", dirName)
+}
+
+// ensure file has suitable permissions if it exists
+func (spec *FixSpec) FixFile(fileName string) {
+	info, err := statNoSymlinks(fileName)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			spec.complainAndExit(err, fileName, spec.FileMode)
+		}
+		return
+	}
+	if info.IsDir() {
+		err = fmt.Errorf("Should not be a directory: '%s'", fileName)
+		spec.complainAndExit(err, fileName, spec.FileMode)
+	}
+	trace.SpamDebug("chown: ", fileName, spec.UserId, spec.GroupId)
+	err = os.Chown(fileName, spec.UserId, spec.GroupId)
+	if err != nil {
+		spec.ensureWritableFile(fileName)
+		return
+	}
+	trace.SpamDebug("chmod: ", fileName, spec.FileMode)
+	err = os.Chmod(fileName, spec.FileMode)
+	if err != nil {
+		spec.ensureWritableFile(fileName)
+	}
+}
+
+func (spec *FixSpec) ensureWritableFile(fileName string) {
+	f, err := os.OpenFile(fileName, os.O_APPEND|os.O_RDWR, spec.FileMode)
+	if err == nil {
+		f.Close()
+		return
+	}
+	trace.Warning(err, "- will try to remove this file")
+	err = os.Remove(fileName)
+	if err != nil {
+		trace.Warning("Could neither write to nor remove '" + fileName + "'")
+		spec.complainAndExit(err, fileName, spec.FileMode)
+	}
+}
+
+func (spec *FixSpec) ensureWritableDir(dirName string) {
+	tmpFile, err := os.CreateTemp(dirName, "tmp.probe.*.tmp")
+	if err != nil {
+		trace.Warning("Could not create a file in directory '" + dirName + "'")
+		spec.complainAndExit(err, dirName, spec.DirMode)
+	}
+	tmpFile.Close()
+	err = os.Remove(tmpFile.Name())
+	if err != nil {
+		spec.complainAndExit(err, dirName, spec.DirMode)
+	}
+}
+
+func (spec *FixSpec) complainAndExit(got error, fn string, wanted os.FileMode) {
+	trace.Warning("problem:", got)
+	currentUser, _ := user.Current()
+	trace.Warning("Currently running as user:", currentUser.Username)
+	trace.Warning("Wanted", fn, "to be owned by user id:", spec.UserId)
+	trace.Warning("Wanted", fn, "to have group id:", spec.GroupId)
+	trace.Warning("Wanted", fn, "to have permissions:", wanted)
+	trace.Warning("current status of", fn, "is:")
+	out, _ := BackTicksWithStderr.Run("stat", "--", fn)
+	trace.Warning(out)
+	trace.Warning("this is a fatal error!")
+	ExitErr(got)
+}
diff --git a/client/go/internal/osutil/fix_fs_test.go b/client/go/internal/osutil/fix_fs_test.go
new file mode 100644
index 00000000000..792986d7996
--- /dev/null
+++ b/client/go/internal/osutil/fix_fs_test.go
@@ -0,0 +1,144 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package osutil
+
+import (
+	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vespa-engine/vespa/client/go/internal/admin/trace"
+	"github.com/vespa-engine/vespa/client/go/internal/ioutil"
+)
+
+func setup(t *testing.T) string {
+	tt := t.TempDir()
+	tmpDir, _ := filepath.EvalSymlinks(tt)
+	err := os.MkdirAll(tmpDir+"/a", 0755)
+	assert.Nil(t, err)
+	err = os.MkdirAll(tmpDir+"/a/bad", 0)
+	assert.Nil(t, err)
+	err = os.WriteFile(tmpDir+"/a/f1", []byte{10}, 0644)
+	assert.Nil(t, err)
+	err = os.WriteFile(tmpDir+"/a/f2", []byte{10}, 0111)
+	return tmpDir
+}
+
+func testFixSpec(t *testing.T, spec FixSpec) {
+	tmpDir := setup(t)
+	spec.FixDir(tmpDir + "/a")
+	spec.FixDir(tmpDir + "/b")
+	spec.FixDir(tmpDir + "/a/bad")
+	spec.FixDir(tmpDir + "/a/bad/ok")
+	spec.FixFile(tmpDir + "/a/f1")
+	spec.FixFile(tmpDir + "/a/f2")
+	spec.FixFile(tmpDir + "/a/f3")
+	spec.FixFile(tmpDir + "/b/f4")
+	spec.FixFile(tmpDir + "/a/bad/f5")
+	assert.Equal(t, true, ioutil.IsDir(tmpDir+"/a"))
+	assert.Equal(t, true, ioutil.IsDir(tmpDir+"/b"))
+	assert.Equal(t, true, ioutil.IsDir(tmpDir+"/a/bad"))
+	assert.Equal(t, true, ioutil.IsDir(tmpDir+"/a/bad/ok"))
+	assert.Equal(t, true, ioutil.IsFile(tmpDir+"/a/f1"))
+	assert.Equal(t, true, ioutil.IsFile(tmpDir+"/a/f2"))
+	assert.Equal(t, false, ioutil.IsFile(tmpDir+"/a/f3"))
+	assert.Equal(t, false, ioutil.IsFile(tmpDir+"/b/f4"))
+	assert.Equal(t, false, ioutil.IsFile(tmpDir+"/a/bad/f5"))
+
+	info, err := os.Stat(tmpDir + "/a")
+	assert.Nil(t, err)
+	assert.Equal(t, true, info.IsDir())
+	assert.Equal(t, 0755, int(info.Mode())&0777)
+
+	info, err = os.Stat(tmpDir + "/b")
+	assert.Nil(t, err)
+	assert.Equal(t, true, info.IsDir())
+	assert.Equal(t, 0755, int(info.Mode())&0777)
+
+	info, err = os.Stat(tmpDir + "/a/bad")
+	assert.Nil(t, err)
+	assert.Equal(t, true, info.IsDir())
+	assert.Equal(t, 0755, int(info.Mode())&0777)
+
+	info, err = os.Stat(tmpDir + "/a/bad/ok")
+	assert.Nil(t, err)
+	assert.Equal(t, true, info.IsDir())
+	assert.Equal(t, 0755, int(info.Mode())&0777)
+
+	info, err = os.Stat(tmpDir + "/a/f1")
+	assert.Nil(t, err)
+	assert.Equal(t, false, info.IsDir())
+	assert.Equal(t, 0644, int(info.Mode())&0777)
+
+	info, err = os.Stat(tmpDir + "/a/f2")
+	assert.Nil(t, err)
+	assert.Equal(t, false, info.IsDir())
+	assert.Equal(t, 0644, int(info.Mode())&0777)
+}
+
+func TestSimpleFixes(t *testing.T) {
+	testFixSpec(t, NewFixSpec())
+}
+
+func TestSuperUserOnly(t *testing.T) {
+	trace.AdjustVerbosity(0)
+	var userId int = -1
+	var groupId int = -1
+	if os.Getuid() != 0 {
+		trace.Trace("skip TestSuperUserOnly, uid != 0")
+		return
+	}
+	u, err := user.Current()
+	if u.Username != "root" {
+		trace.Trace("skip TestSuperUserOnly, user != root")
+		return
+	}
+	u, err = user.Lookup("nobody")
+	if err != nil {
+		trace.Trace("skip TestSuperUserOnly, user nobody was not found")
+		return
+	}
+	userId, err = strconv.Atoi(u.Uid)
+	if err != nil || userId < 1 {
+		trace.Trace("skip TestSuperUserOnly, user ID of nobody was not found")
+		return
+	}
+	g, err := user.LookupGroup("users")
+	if err == nil {
+		groupId, _ = strconv.Atoi(g.Gid)
+	}
+	fixSpec := NewFixSpec()
+	fixSpec.UserId = userId
+	if groupId > 0 {
+		fixSpec.GroupId = groupId
+	}
+	testFixSpec(t, fixSpec)
+}
+
+func expectSimplePanic() {
+	if r := recover(); r != nil {
+		if jee, ok := r.(*ExitError); ok {
+			trace.Trace("got as expected:", jee)
+			return
+		}
+		panic(r)
+	}
+}
+
+func TestFailedFixdir(t *testing.T) {
+	tmpDir := setup(t)
+	spec := NewFixSpec()
+	defer expectSimplePanic()
+	spec.FixDir(tmpDir + "/a/f1")
+	assert.Equal(t, "", "should not be reached")
+}
+
+func TestFailedFixfile(t *testing.T) {
+	tmpDir := setup(t)
+	spec := NewFixSpec()
+	defer expectSimplePanic()
+	spec.FixFile(tmpDir + "/a")
+	assert.Equal(t, "", "should not be reached")
+}
diff --git a/client/go/internal/osutil/just_exit.go b/client/go/internal/osutil/just_exit.go
new file mode 100644
index 00000000000..5ad85ec9ceb
--- /dev/null
+++ b/client/go/internal/osutil/just_exit.go
@@ -0,0 +1,50 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package osutil
+
+import (
+	"fmt"
+
+	"github.com/vespa-engine/vespa/client/go/internal/admin/trace"
+)
+
+type ExitError struct {
+	err error
+	msg string
+}
+
+func (j *ExitError) String() string {
+	if j.err != nil {
+		if j.msg == "" {
+			return j.err.Error()
+		}
+		return fmt.Sprintf("%s: %s", j.msg, j.err.Error())
+	}
+	if j.msg == "" {
+		panic(j)
+	}
+	return j.msg
+}
+
+func (j *ExitError) Error() string {
+	return j.String()
+}
+
+func ExitMsg(message string) {
+	trace.Trace("just exit with message")
+	j := ExitError{
+		err: nil,
+		msg: message,
+	}
+	panic(&j)
+}
+
+func ExitErr(e error) {
+	trace.Trace("just exit with error")
+	j := ExitError{
+		err: e,
+		msg: "",
+	}
+	panic(&j)
+}
diff --git a/client/go/internal/osutil/run_cmd.go b/client/go/internal/osutil/run_cmd.go
new file mode 100644
index 00000000000..ca0d621f9f9
--- /dev/null
+++ b/client/go/internal/osutil/run_cmd.go
@@ -0,0 +1,42 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package osutil
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/vespa-engine/vespa/client/go/internal/admin/trace"
+)
+
+type BackTicks int
+
+const (
+	BackTicksWithStderr BackTicks = iota
+	BackTicksIgnoreStderr
+	BackTicksForwardStderr
+	SystemCommand
+)
+
+func (b BackTicks) Run(program string, args ...string) (string, error) {
+	cmd := exec.Command(program, args...)
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	switch b {
+	case BackTicksWithStderr:
+		cmd.Stderr = &out
+	case BackTicksIgnoreStderr:
+		cmd.Stderr = nil
+	case BackTicksForwardStderr:
+		cmd.Stderr = os.Stderr
+	case SystemCommand:
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	}
+	trace.Debug("running command:", program, strings.Join(args, " "))
+	err := cmd.Run()
+	return out.String(), err
+}
diff --git a/client/go/internal/osutil/setrlimit.go b/client/go/internal/osutil/setrlimit.go
new file mode 100644
index 00000000000..6bc6d68af3e
--- /dev/null
+++ b/client/go/internal/osutil/setrlimit.go
@@ -0,0 +1,72 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+//go:build !windows
+
+package osutil
+
+import (
+	"os"
+	"strconv"
+
+	"github.com/vespa-engine/vespa/client/go/internal/admin/trace"
+	"golang.org/x/sys/unix"
+)
+
+type ResourceId int
+
+const (
+	RLIMIT_CORE   ResourceId = unix.RLIMIT_CORE
+	RLIMIT_NOFILE ResourceId = unix.RLIMIT_NOFILE
+	RLIMIT_NPROC  ResourceId = unix.RLIMIT_NPROC
+	NO_RLIMIT     uint64     = ^uint64(0)
+)
+
+func (rid ResourceId) String() string {
+	switch rid {
+	case RLIMIT_CORE:
+		return "core file size"
+	case RLIMIT_NOFILE:
+		return "open files"
+	case RLIMIT_NPROC:
+		return "max user processes"
+	}
+	return "unknown resource id"
+}
+
+func readableLimit(val uint64) string {
+	if val == NO_RLIMIT {
+		return "unlimited"
+	}
+	return strconv.FormatUint(val, 10)
+}
+
+func SetResourceLimit(resource ResourceId, newVal uint64) {
+	trace.Debug("Wanted", newVal, "as limit for", resource.String())
+	var current unix.Rlimit
+	err := unix.Getrlimit(int(resource), &current)
+	if err != nil {
+		trace.Warning("Could not get current resource limit:", err)
+		return
+	}
+	wanted := current
+	if current.Max < newVal {
+		if os.Getuid() == 0 {
+			wanted.Max = newVal
+		} else if newVal > current.Max {
+			trace.Warning(
+				"Wanted", newVal,
+				"as limit for", resource.String(),
+				"but cannot exceed current hard limit:", current.Max)
+			newVal = current.Max
+		}
+	}
+	wanted.Cur = newVal
+	err = unix.Setrlimit(int(resource), &wanted)
+	if err != nil {
+		trace.Trace("Failed setting limit for", resource, ":", err)
+	} else {
+		trace.Debug("Resource limit", resource, "was:", readableLimit(current.Cur), "/", readableLimit(current.Max))
+		_ = unix.Getrlimit(int(resource), &current)
+		trace.Trace("Resource limit", resource, "adjusted to:", readableLimit(current.Cur), "/", readableLimit(current.Max))
+	}
+}
diff --git a/client/go/internal/osutil/setrlimit_windows.go b/client/go/internal/osutil/setrlimit_windows.go
new file mode 100644
index 00000000000..e61233ba9e6
--- /dev/null
+++ b/client/go/internal/osutil/setrlimit_windows.go
@@ -0,0 +1,18 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+//go:build windows
+
+package osutil
+
+type ResourceId int
+
+const (
+	RLIMIT_CORE ResourceId = iota
+	RLIMIT_NOFILE
+	RLIMIT_NPROC
+	NO_RLIMIT uint64 = ^uint64(0)
+)
+
+func SetResourceLimit(resource ResourceId, max uint64) {
+	// nop
+}
diff --git a/client/go/internal/osutil/tune_logctl.go b/client/go/internal/osutil/tune_logctl.go
new file mode 100644
index 00000000000..f68259170c7
--- /dev/null
+++ b/client/go/internal/osutil/tune_logctl.go
@@ -0,0 +1,13 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package osutil
+
+func TuneLogging(serviceName, component, settings string) bool {
+	arg := serviceName
+	if component != "" {
+		arg = serviceName + ":" + component
+	}
+	_, err := BackTicksIgnoreStderr.Run("vespa-logctl", "-c", arg, settings)
+	return err == nil
+}
diff --git a/client/go/internal/osutil/tuning.go b/client/go/internal/osutil/tuning.go
new file mode 100644
index 00000000000..8e9b894e8ae
--- /dev/null
+++ b/client/go/internal/osutil/tuning.go
@@ -0,0 +1,59 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package osutil
+
+import (
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/vespa-engine/vespa/client/go/internal/admin/envvars"
+	"github.com/vespa-engine/vespa/client/go/internal/admin/trace"
+)
+
+func OptionallyReduceTimerFrequency() {
+	if os.Getenv(envvars.VESPA_TIMER_HZ) == "" {
+		backticks := BackTicksIgnoreStderr
+		uname, _ := backticks.Run("uname", "-r")
+		if strings.Contains(uname, "linuxkit") {
+			setTimerHZ("Docker on macOS detected.", "100")
+		} else {
+			virt, _ := backticks.Run("systemd-detect-virt", "--vm")
+			if strings.TrimSpace(virt) == "qemu" {
+				setTimerHZ("QEMU virtualization detected.", "100")
+			}
+		}
+	}
+}
+
+func setTimerHZ(description, timerHZ string) {
+	if os.Getenv(envvars.VESPA_TIMER_HZ) == timerHZ {
+		return
+	}
+	trace.Trace(
+		description,
+		"Reducing base frequency from 1000hz to "+timerHZ+"hz due to high cost of sampling time.",
+		"This will reduce timeout accuracy.")
+	os.Setenv(envvars.VESPA_TIMER_HZ, timerHZ)
+}
+
+func TuneResourceLimits() {
+	var numfiles uint64 = 262144
+	var numprocs uint64 = 409600
+	if env := os.Getenv(envvars.FILE_DESCRIPTOR_LIMIT); env != "" {
+		n, err := strconv.Atoi(env)
+		if err != nil {
+			numfiles = uint64(n)
+		}
+	}
+	if env := os.Getenv(envvars.NUM_PROCESSES_LIMIT); env != "" {
+		n, err := strconv.Atoi(env)
+		if err != nil {
+			numprocs = uint64(n)
+		}
+	}
+	SetResourceLimit(RLIMIT_CORE, NO_RLIMIT)
+	SetResourceLimit(RLIMIT_NOFILE, numfiles)
+	SetResourceLimit(RLIMIT_NPROC, numprocs)
+}