diff options
Diffstat (limited to 'client')
| -rw-r--r-- | client/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | client/go/script-utils/main.go | 30 | ||||
| -rw-r--r-- | client/go/vespa/detect_hostname.go | 135 | ||||
| -rw-r--r-- | client/go/vespa/detect_hostname_test.go | 30 | ||||
| -rw-r--r-- | client/go/vespa/load_env.go | 164 | ||||
| -rw-r--r-- | client/go/vespa/load_env_test.go | 65 |
6 files changed, 416 insertions, 9 deletions
diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index c87e9855fe9..07361e4b2eb 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -13,3 +13,4 @@ add_custom_target(vespalog_logfmt ALL DEPENDS ${GODIR}/bin/vespa-logfmt) install(PROGRAMS ${GODIR}/bin/vespa-logfmt DESTINATION bin) install(PROGRAMS ${GODIR}/bin/vespa-deploy DESTINATION bin) +install(PROGRAMS ${GODIR}/bin/script-utils DESTINATION libexec/vespa) diff --git a/client/go/script-utils/main.go b/client/go/script-utils/main.go new file mode 100644 index 00000000000..a7160691a5d --- /dev/null +++ b/client/go/script-utils/main.go @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package main + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/vespa" +) + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "actions: export-env, ipv6-only") + return + } + switch os.Args[1] { + case "export-env": + vespa.ExportDefaultEnvToSh() + case "ipv6-only": + if vespa.HasOnlyIpV6() { + os.Exit(0) + } else { + os.Exit(1) + } + default: + fmt.Fprintf(os.Stderr, "unknown action '%s'\n", os.Args[1]) + } +} diff --git a/client/go/vespa/detect_hostname.go b/client/go/vespa/detect_hostname.go new file mode 100644 index 00000000000..80f5df2c866 --- /dev/null +++ b/client/go/vespa/detect_hostname.go @@ -0,0 +1,135 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package vespa + +import ( + "fmt" + "net" + "os" + "strings" +) + +// detect if this host is IPv6-only, in which case we want to pass +// the flag "-Djava.net.preferIPv6Addresses=true" to any java command +func HasOnlyIpV6() bool { + hostname, err := FindOurHostname() + if hostname == "" || err != nil { + return false + } + foundV4 := false + foundV6 := false + ipAddrs, err := net.LookupIP(hostname) + if err != nil { + return false + } + for _, addr := range ipAddrs { + switch { + case addr.IsLoopback(): + // skip + case addr.To4() != nil: + foundV4 = true + case addr.To16() != nil: + foundV6 = true + } + } + return foundV6 && !foundV4 +} + +// Find a good name for the host we're running on. +// We need something that *other* hosts can use for connnecting back +// to our services, preferably the canonical DNS name. +// If automatic detection fails, "localhost" will be returned, so +// single-node setups still have a good chance of working. +// Use the enviroment variable VESPA_HOSTNAME to override. +func FindOurHostname() (string, error) { + env := os.Getenv("VESPA_HOSTNAME") + if env != "" { + // assumes: env var is already validated and OK + return env, nil + } + name, err := os.Hostname() + if err != nil { + return findOurHostnameFrom("localhost") + } + name, err = findOurHostnameFrom(name) + if strings.HasSuffix(name, ".") { + name = name[:len(name)-1] + } + return name, err +} + +func validateHostname(name string) bool { + myIpAddresses := make(map[string]bool) + interfaceAddrs, _ := net.InterfaceAddrs() + for _, ifAddr := range interfaceAddrs { + // note: ifAddr.String() is typically "127.0.0.1/8" + if ipnet, ok := ifAddr.(*net.IPNet); ok { + myIpAddresses[ipnet.IP.String()] = true + } + } + ipAddrs, _ := net.LookupIP(name) + someGood := false + for _, addr := range ipAddrs { + if len(myIpAddresses) == 0 { + // no validation possible, assume OK + return true + } + if myIpAddresses[addr.String()] { + someGood = true + } else { + return false + } + } + return someGood +} + +func findOurHostnameFrom(name string) (string, error) { + if strings.Contains(name, ".") && validateHostname(name) { + // it's all good + return name, nil + } + possibles := make([]string, 0, 5) + if name != "" { + ipAddrs, _ := net.LookupIP(name) + for _, addr := range ipAddrs { + switch { + case addr.IsLoopback(): + // skip + case addr.To4() != nil || addr.To16() != nil: + reverseNames, _ := net.LookupAddr(addr.String()) + possibles = append(possibles, reverseNames...) + } + } + } + interfaceAddrs, _ := net.InterfaceAddrs() + for _, ifAddr := range interfaceAddrs { + if ipnet, ok := ifAddr.(*net.IPNet); ok { + ip := ipnet.IP + if ip == nil || ip.IsLoopback() { + continue + } + reverseNames, _ := net.LookupAddr(ip.String()) + possibles = append(possibles, reverseNames...) + } + } + // look for valid possible starting with the given name + for _, poss := range possibles { + if strings.HasPrefix(poss, name+".") && validateHostname(poss) { + return poss, nil + } + } + // look for valid possible + for _, poss := range possibles { + if strings.Contains(poss, ".") && validateHostname(poss) { + return poss, nil + } + } + // look for any valid possible + for _, poss := range possibles { + if validateHostname(poss) { + return poss, nil + } + } + return "localhost", fmt.Errorf("fallback to localhost, os.Hostname '%s'", name) +} diff --git a/client/go/vespa/detect_hostname_test.go b/client/go/vespa/detect_hostname_test.go new file mode 100644 index 00000000000..398626b708f --- /dev/null +++ b/client/go/vespa/detect_hostname_test.go @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package vespa + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDetectHostname(t *testing.T) { + t.Setenv("VESPA_HOSTNAME", "foo.bar") + got, err := FindOurHostname() + assert.Equal(t, "foo.bar", got) + os.Unsetenv("VESPA_HOSTNAME") + got, err = findOurHostnameFrom("bar.foo.123") + fmt.Fprintln(os.Stderr, "findOurHostname from bar.foo.123 returns:", got, "with error:", err) + assert.NotEqual(t, "", got) + parts := strings.Split(got, ".") + if len(parts) > 1 { + expanded, err2 := findOurHostnameFrom(parts[0]) + fmt.Fprintln(os.Stderr, "findOurHostname from", parts[0], "returns:", expanded, "with error:", err2) + assert.Equal(t, got, expanded) + } + got, err = FindOurHostname() + assert.NotEqual(t, "", got) + fmt.Fprintln(os.Stderr, "FindOurHostname() returns:", got, "with error:", err) +} diff --git a/client/go/vespa/load_env.go b/client/go/vespa/load_env.go index a0c127ca920..8eb7c841235 100644 --- a/client/go/vespa/load_env.go +++ b/client/go/vespa/load_env.go @@ -8,11 +8,77 @@ import ( "bufio" "fmt" "os" + "os/user" "strings" ) // backwards-compatible parsing of default-env.txt func LoadDefaultEnv() error { + return loadDefaultEnvTo(new(osEnvReceiver)) +} + +// parse default-env.txt, then dump export statements for "sh" to stdout +func ExportDefaultEnvToSh() error { + holder := newShellEnvExporter() + err := loadDefaultEnvTo(holder) + holder.dump() + return err +} + +// Which user should vespa services run as? If current user is root, +// we want to change to some non-privileged user. +// Should be run after LoadDefaultEnv() which possibly loads VESPA_USER +func FindVespaUser() string { + uName := os.Getenv("VESPA_USER") + if uName != "" { + // no check here, assume valid + return uName + } + if os.Getuid() == 0 { + u, err := user.Lookup("vespa") + if err == nil { + uName = u.Username + } else { + u, err = user.Lookup("nobody") + if err == nil { + uName = u.Username + } + } + } + if uName == "" { + u, err := user.Current() + if err == nil { + uName = u.Username + } + } + if uName != "" { + os.Setenv("VESPA_USER", uName) + } + return uName +} + +type loadEnvReceiver interface { + fallbackVar(varName, varVal string) + overrideVar(varName, varVal string) + unsetVar(varName string) +} + +type osEnvReceiver struct { +} + +func (p *osEnvReceiver) fallbackVar(varName, varVal string) { + if os.Getenv(varName) == "" { + os.Setenv(varName, varVal) + } +} +func (p *osEnvReceiver) overrideVar(varName, varVal string) { + os.Setenv(varName, varVal) +} +func (p *osEnvReceiver) unsetVar(varName string) { + os.Unsetenv(varName) +} + +func loadDefaultEnvTo(r loadEnvReceiver) error { const defEnvTxt = "/conf/vespa/default-env.txt" vespaHome := FindHome() f, err := os.Open(vespaHome + defEnvTxt) @@ -42,13 +108,11 @@ func LoadDefaultEnv() error { } switch action { case "override": - os.Setenv(varName, varVal) + r.overrideVar(varName, varVal) case "fallback": - if os.Getenv(varName) == "" { - os.Setenv(varName, varVal) - } + r.fallbackVar(varName, varVal) case "unset": - os.Unsetenv(varName) + r.unsetVar(varName) default: err = fmt.Errorf("unknown action '%s'", action) } @@ -107,3 +171,93 @@ func isValidShellVariableName(s string) bool { } return len(s) > 0 } + +type shellEnvExporter struct { + exportVars map[string]string + unsetVars map[string]string +} + +func newShellEnvExporter() *shellEnvExporter { + return &shellEnvExporter{ + exportVars: make(map[string]string), + unsetVars: make(map[string]string), + } +} +func (p *shellEnvExporter) fallbackVar(varName, varVal string) { + if os.Getenv(varName) == "" || p.unsetVars[varName] != "" { + delete(p.unsetVars, varName) + p.exportVars[varName] = shellQuote(varVal) + } +} +func (p *shellEnvExporter) overrideVar(varName, varVal string) { + delete(p.unsetVars, varName) + p.exportVars[varName] = shellQuote(varVal) +} +func (p *shellEnvExporter) unsetVar(varName string) { + delete(p.exportVars, varName) + p.unsetVars[varName] = "unset" +} + +func shellQuote(s string) string { + l := 0 + nq := false + for _, ch := range s { + switch { + case (ch >= 'A' && ch <= 'Z') || + (ch >= 'a' && ch <= 'z') || + (ch >= '0' && ch <= '9'): + l++ + case ch == '_' || ch == ' ': + l++ + nq = true + case ch == '\'' || ch == '\\': + l = l + 4 + nq = true + default: + l++ + nq = true + } + } + if nq { + l = l + 2 + } + res := make([]rune, l) + i := 0 + if nq { + res[i] = '\'' + i++ + } + for _, ch := range s { + if ch == '\'' || ch == '\\' { + res[i] = '\'' + i++ + res[i] = '\\' + i++ + res[i] = ch + i++ + res[i] = '\'' + } else { + res[i] = ch + } + i++ + } + if nq { + res[i] = '\'' + i++ + } + if i != l { + err := fmt.Errorf("expected length %d but was %d", l, i) + panic(err) + } + return string(res) +} + +func (p *shellEnvExporter) dump() { + for vn, vv := range p.exportVars { + fmt.Printf("%s=%s\n", vn, vv) + fmt.Printf("export %s\n", vn) + } + for vn, _ := range p.unsetVars { + fmt.Printf("unset %s\n", vn) + } +} diff --git a/client/go/vespa/load_env_test.go b/client/go/vespa/load_env_test.go index c5b42cae161..41373f7ab82 100644 --- a/client/go/vespa/load_env_test.go +++ b/client/go/vespa/load_env_test.go @@ -2,6 +2,7 @@ package vespa import ( + "fmt" "os" "testing" @@ -15,15 +16,15 @@ func setup(t *testing.T, contents string) { envf := cdir + "/default-env.txt" err := os.MkdirAll(cdir, 0755) assert.Nil(t, err) - os.Setenv("VESPA_HOME", vdir) + t.Setenv("VESPA_HOME", vdir) err = os.WriteFile(envf, []byte(contents), 0644) assert.Nil(t, err) } func TestLoadEnvSimple(t *testing.T) { - os.Setenv("VESPA_FOO", "was foo") - os.Setenv("VESPA_BAR", "was bar") - os.Setenv("VESPA_FOOBAR", "foobar") + t.Setenv("VESPA_FOO", "was foo") + t.Setenv("VESPA_BAR", "was bar") + t.Setenv("VESPA_FOOBAR", "foobar") os.Unsetenv("VESPA_QUUX") setup(t, ` # vespa env vars file @@ -98,3 +99,59 @@ override VESPA_V2 v2 assert.NotNil(t, err) assert.Equal(t, err.Error(), "Not a valid environment variable name: '.A'") } + +func TestFindUser(t *testing.T) { + u := FindVespaUser() + if u == "" { + fmt.Fprintln(os.Stderr, "WARNING: empty result from FindVespaUser()") + } else { + fmt.Fprintln(os.Stderr, "INFO: result from FindVespaUser() is", u) + assert.Equal(t, u, os.Getenv("VESPA_USER")) + } + setup(t, ` +override VESPA_USER unprivuser +`) + LoadDefaultEnv() + u = FindVespaUser() + assert.Equal(t, "unprivuser", u) +} + +func TestExportEnv(t *testing.T) { + t.Setenv("VESPA_FOO", "was foo") + t.Setenv("VESPA_BAR", "was bar") + t.Setenv("VESPA_FOOBAR", "foobar") + t.Setenv("VESPA_BARFOO", "was barfoo") + os.Unsetenv("VESPA_QUUX") + setup(t, ` +# vespa env vars file +override VESPA_FOO "newFoo1" + +fallback VESPA_BAR "new bar" +fallback VESPA_QUUX "new quux" + +unset VESPA_FOOBAR +unset VESPA_BARFOO +fallback VESPA_BARFOO new'b<a>r'foo +override XYZ xyz +unset XYZ +`) + holder := newShellEnvExporter() + err := loadDefaultEnvTo(holder) + assert.Nil(t, err) + // new values: + assert.Equal(t, "newFoo1", holder.exportVars["VESPA_FOO"]) + assert.Equal(t, "", holder.exportVars["VESPA_BAR"]) + assert.Equal(t, "'new quux'", holder.exportVars["VESPA_QUUX"]) + assert.Equal(t, `'new'\''b<a>r'\''foo'`, holder.exportVars["VESPA_BARFOO"]) + // unsets: + assert.Equal(t, "", holder.exportVars["VESPA_FOOBAR"]) + assert.Equal(t, "unset", holder.unsetVars["VESPA_FOOBAR"]) + assert.Equal(t, "", holder.exportVars["XYZ"]) + assert.Equal(t, "unset", holder.unsetVars["XYZ"]) + // nothing extra allowed: + assert.Equal(t, 3, len(holder.exportVars)) + assert.Equal(t, 2, len(holder.unsetVars)) + // run it + err = ExportDefaultEnvToSh() + assert.Nil(t, err) +} |