diff options
author | Martin Polden <mpolden@mpolden.no> | 2023-02-03 15:20:23 +0100 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2023-02-03 15:35:25 +0100 |
commit | e1e94812425a487069bf33f781bec987e9e49874 (patch) | |
tree | 4a892c3b5c0a7dee2cb76f9971e538cb4aba8a16 /client/go/internal/admin | |
parent | a08ae588d6035b69f0961dff596fc871fd1c4e58 (diff) |
Re-organize Go code
Diffstat (limited to 'client/go/internal/admin')
111 files changed, 6495 insertions, 0 deletions
diff --git a/client/go/internal/admin/defaults/defaults.go b/client/go/internal/admin/defaults/defaults.go new file mode 100644 index 00000000000..10dfe5cb16b --- /dev/null +++ b/client/go/internal/admin/defaults/defaults.go @@ -0,0 +1,217 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package defaults + +import ( + "fmt" + "os" + "strconv" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +const ( + DEFAULT_VESPA_HOME = "/opt/vespa" + DEFAULT_VESPA_USER = "vespa" + DEFAULT_VESPA_HOST = "localhost" + + DEFAULT_VESPA_PORT_BASE = 19000 + CONFIGSERVER_RPC_PORT_OFFSET = 70 + CONFIGPROXY_RPC_PORT_OFFSET = 90 + DEFAULT_WEB_SERVICE_PORT = 8080 +) + +// Compute the path prefix where Vespa files will live. +// Note: does not end with "/" +func VespaHome() string { + if env := os.Getenv(envvars.VESPA_HOME); env != "" { + return env + } + return DEFAULT_VESPA_HOME +} + +func UnderVespaHome(p string) string { + if strings.HasPrefix(p, "/") || strings.HasPrefix(p, "./") { + return p + } + return fmt.Sprintf("%s/%s", VespaHome(), p) +} + +// Compute the user name to own directories and run processes. +func VespaUser() string { + if env := os.Getenv(envvars.VESPA_USER); env != "" { + return env + } + return DEFAULT_VESPA_USER +} + +// Compute the host name that identifies myself. +// Detection of the hostname is now done before starting any Vespa +// programs and provided in the environment variable VESPA_HOSTNAME; +// if that variable isn't set a default of "localhost" is always returned. +func VespaHostname() string { + if env := os.Getenv(envvars.VESPA_HOSTNAME); env != "" { + return env + } + return DEFAULT_VESPA_HOST +} + +func VespaLogFile() string { + if env := os.Getenv(envvars.VESPA_LOG_TARGET); env != "" { + if strings.HasPrefix(env, "file:") { + return strings.TrimPrefix(env, "file:") + } + } + return UnderVespaHome("logs/vespa/vespa.log") +} + +// Compute the port number where the Vespa webservice +// container should be available. +func VespaContainerWebServicePort() int { + p := getNumFromEnv(envvars.VESPA_WEB_SERVICE_PORT) + if p > 0 { + trace.Debug(envvars.VESPA_WEB_SERVICE_PORT, p) + return p + } + return DEFAULT_WEB_SERVICE_PORT +} + +// Compute the base for port numbers where the Vespa services should listen. +func VespaPortBase() int { + p := getNumFromEnv(envvars.VESPA_PORT_BASE) + if p > 0 { + trace.Debug(envvars.VESPA_PORT_BASE, p) + return p + } + return DEFAULT_VESPA_PORT_BASE +} + +// Find the hostnames of configservers that are configured. +func VespaConfigserverHosts() []string { + parts := splitVespaConfigservers() + rv := make([]string, len(parts)) + for idx, part := range parts { + if colon := strings.Index(part, ":"); colon > 0 { + rv[idx] = part[0:colon] + } else { + rv[idx] = part + } + trace.Debug("config server host:", rv[idx]) + } + return rv +} + +func findConfigserverHttpPort() int { + return findConfigserverRpcPort() + 1 +} + +// Find the RPC addresses to configservers that are configured. +// Returns a list of RPC specs in the format tcp/{hostname}:{portnumber} +func VespaConfigserverRpcAddrs() []string { + parts := splitVespaConfigservers() + rv := make([]string, len(parts)) + for idx, part := range parts { + if colon := strings.Index(part, ":"); colon > 0 { + rv[idx] = fmt.Sprintf("tcp/%s", part) + } else { + rv[idx] = fmt.Sprintf("tcp/%s:%d", part, findConfigserverRpcPort()) + } + trace.Debug("config server rpc addr:", rv[idx]) + } + return rv +} + +// Find the URLs to the REST api on configservers +// Returns a list of URLS in the format http://{hostname}:{portnumber}/ +func VespaConfigserverRestUrls() []string { + parts := splitVespaConfigservers() + rv := make([]string, len(parts)) + for idx, hostnm := range parts { + port := findConfigserverHttpPort() + if colon := strings.Index(hostnm, ":"); colon > 0 { + p, err := strconv.Atoi(hostnm[colon+1:]) + if err == nil && p > 0 { + port = p + 1 + } + hostnm = hostnm[:colon] + } + rv[idx] = fmt.Sprintf("http://%s:%d", hostnm, port) + trace.Debug("config server rest url:", rv[idx]) + } + return rv +} + +// Find the RPC address to the local config proxy +// Returns one RPC spec in the format tcp/{hostname}:{portnumber} +func VespaConfigProxyRpcAddr() string { + return fmt.Sprintf("tcp/localhost:%d", findConfigproxyRpcPort()) +} + +// Get the RPC addresses to all known config sources +// Returns same as vespaConfigProxyRpcAddr + vespaConfigserverRpcAddrs +func VespaConfigSourcesRpcAddrs() []string { + cs := VespaConfigserverRpcAddrs() + rv := make([]string, 0, len(cs)+1) + rv = append(rv, VespaConfigProxyRpcAddr()) + for _, addr := range cs { + rv = append(rv, addr) + } + return rv +} + +func splitVespaConfigservers() []string { + env := os.Getenv(envvars.VESPA_CONFIGSERVERS) + if env == "" { + env = os.Getenv(envvars.ADDR_CONFIGSERVER) + } + parts := make([]string, 0, 3) + for { + idx := strings.IndexAny(env, " ,") + if idx < 0 { + break + } + if idx > 0 { + parts = append(parts, env[:idx]) + } + env = env[idx+1:] + } + if env != "" { + parts = append(parts, env) + } + if len(parts) == 0 { + parts = append(parts, "localhost") + } + return parts +} + +func findConfigproxyRpcPort() int { + p := getNumFromEnv(envvars.CONFIGPROXY_RPC_PORT) + if p > 0 { + return p + } + return VespaPortBase() + CONFIGPROXY_RPC_PORT_OFFSET +} + +func findConfigserverRpcPort() int { + p := getNumFromEnv(envvars.CONFIGSERVER_RPC_PORT) + if p > 0 { + trace.Debug(envvars.CONFIGSERVER_RPC_PORT, p) + return p + } + return VespaPortBase() + CONFIGSERVER_RPC_PORT_OFFSET +} + +func getNumFromEnv(vn string) int { + env := os.Getenv(vn) + if env != "" { + p, err := strconv.Atoi(env) + if err == nil { + return p + } + trace.Debug("env var", vn, "is:", env, "parse error:", err) + } + return -1 +} diff --git a/client/go/internal/admin/defaults/defaults_test.go b/client/go/internal/admin/defaults/defaults_test.go new file mode 100644 index 00000000000..f900a844285 --- /dev/null +++ b/client/go/internal/admin/defaults/defaults_test.go @@ -0,0 +1,194 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package defaults + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +func setup(t *testing.T) { + t.Setenv("VESPA_HOME", "/home/v/1") + t.Setenv("VESPA_USER", "somebody") + t.Setenv("VESPA_HOSTNAME", "foo.bar.local") + t.Setenv("VESPA_CONFIGSERVERS", "foo1.local, bar2.local, baz3.local") + t.Setenv("VESPA_PORT_BASE", "17000") + t.Setenv("port_configserver_rpc", "") + t.Setenv("port_configproxy_rpc", "") + t.Setenv("VESPA_WEB_SERVICE_PORT", "") +} + +func TestWebServicePort(t *testing.T) { + trace.AdjustVerbosity(1) + setup(t) + ws := VespaContainerWebServicePort() + assert.Equal(t, 8080, ws) + + t.Setenv("VESPA_WEB_SERVICE_PORT", "4488") + ws = VespaContainerWebServicePort() + assert.Equal(t, 4488, ws) +} + +func TestConfigProxyRpcAddr(t *testing.T) { + setup(t) + addr := VespaConfigProxyRpcAddr() + assert.Equal(t, "tcp/localhost:17090", addr) + t.Setenv("VESPA_PORT_BASE", "") + addr = VespaConfigProxyRpcAddr() + assert.Equal(t, "tcp/localhost:19090", addr) + t.Setenv("port_configproxy_rpc", "16066") + addr = VespaConfigProxyRpcAddr() + assert.Equal(t, "tcp/localhost:16066", addr) +} + +func TestConfigSourcesRpcAddrs(t *testing.T) { + setup(t) + cs := VespaConfigSourcesRpcAddrs() + assert.Equal(t, len(cs), 4) + assert.Equal(t, cs[0], "tcp/localhost:17090") + assert.Equal(t, cs[1], "tcp/foo1.local:17070") + t.Setenv("port_configserver_rpc", "12345") + cs = VespaConfigSourcesRpcAddrs() + assert.Equal(t, len(cs), 4) + assert.Equal(t, cs[1], "tcp/foo1.local:12345") +} + +func TestConfigserverHosts(t *testing.T) { + setup(t) + var cs []string + t.Setenv("VESPA_CONFIGSERVERS", "foo.bar") + cs = VespaConfigserverHosts() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "foo.bar") + cs = VespaConfigserverRpcAddrs() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "tcp/foo.bar:17070") + cs = VespaConfigserverRestUrls() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "http://foo.bar:17071") + + t.Setenv("VESPA_CONFIGSERVERS", "foo.bar:18080") + cs = VespaConfigserverHosts() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "foo.bar") + cs = VespaConfigserverRpcAddrs() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "tcp/foo.bar:18080") + cs = VespaConfigserverRestUrls() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "http://foo.bar:18081") + + t.Setenv("VESPA_CONFIGSERVERS", "foo.local, bar.local") + cs = VespaConfigserverHosts() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "foo.local") + assert.Equal(t, cs[1], "bar.local") + cs = VespaConfigserverRpcAddrs() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "tcp/foo.local:17070") + assert.Equal(t, cs[1], "tcp/bar.local:17070") + cs = VespaConfigserverRestUrls() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "http://foo.local:17071") + assert.Equal(t, cs[1], "http://bar.local:17071") + + t.Setenv("VESPA_CONFIGSERVERS", "foo bar") + cs = VespaConfigserverHosts() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "foo") + assert.Equal(t, cs[1], "bar") + cs = VespaConfigserverRpcAddrs() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "tcp/foo:17070") + assert.Equal(t, cs[1], "tcp/bar:17070") + cs = VespaConfigserverRestUrls() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "http://foo:17071") + assert.Equal(t, cs[1], "http://bar:17071") + + t.Setenv("VESPA_CONFIGSERVERS", "foo,bar") + cs = VespaConfigserverHosts() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "foo") + assert.Equal(t, cs[1], "bar") + cs = VespaConfigserverRpcAddrs() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "tcp/foo:17070") + assert.Equal(t, cs[1], "tcp/bar:17070") + cs = VespaConfigserverRestUrls() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "http://foo:17071") + assert.Equal(t, cs[1], "http://bar:17071") + + t.Setenv("VESPA_CONFIGSERVERS", " foo , bar, ") + cs = VespaConfigserverHosts() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "foo") + assert.Equal(t, cs[1], "bar") + cs = VespaConfigserverRpcAddrs() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "tcp/foo:17070") + assert.Equal(t, cs[1], "tcp/bar:17070") + cs = VespaConfigserverRestUrls() + assert.Equal(t, len(cs), 2) + assert.Equal(t, cs[0], "http://foo:17071") + assert.Equal(t, cs[1], "http://bar:17071") + + os.Unsetenv("VESPA_CONFIGSERVERS") + cs = VespaConfigserverHosts() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "localhost") + cs = VespaConfigserverRpcAddrs() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "tcp/localhost:17070") + cs = VespaConfigserverRestUrls() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "http://localhost:17071") + + t.Setenv("VESPA_PORT_BASE", "") + cs = VespaConfigserverRpcAddrs() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "tcp/localhost:19070") +} + +func TestConfigserverHostsWithPortOverride(t *testing.T) { + setup(t) + var cs []string + t.Setenv("VESPA_CONFIGSERVERS", "foo.bar") + t.Setenv("port_configserver_rpc", "12345") + cs = VespaConfigserverRpcAddrs() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "tcp/foo.bar:12345") + cs = VespaConfigserverRestUrls() + assert.Equal(t, len(cs), 1) + assert.Equal(t, cs[0], "http://foo.bar:12346") +} + +func TestUser(t *testing.T) { + setup(t) + user := VespaUser() + assert.Equal(t, "somebody", user) + t.Setenv("VESPA_USER", "") + user = VespaUser() + assert.Equal(t, "vespa", user) +} + +func TestHome(t *testing.T) { + setup(t) + home := VespaHome() + assert.Equal(t, "/home/v/1", home) + t.Setenv("VESPA_HOME", "") + home = VespaHome() + assert.Equal(t, "/opt/vespa", home) +} + +func TestHost(t *testing.T) { + setup(t) + host := VespaHostname() + assert.Equal(t, "foo.bar.local", host) + t.Setenv("VESPA_HOSTNAME", "") + host = VespaHostname() + assert.Equal(t, "localhost", host) +} diff --git a/client/go/internal/admin/envvars/env_vars.go b/client/go/internal/admin/envvars/env_vars.go new file mode 100644 index 00000000000..ad58338254c --- /dev/null +++ b/client/go/internal/admin/envvars/env_vars.go @@ -0,0 +1,89 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package envvars + +// well-known environment variable names + +const ( + ADDR_CONFIGSERVER = "addr_configserver" + CLOUDCONFIG_SERVER_MULTITENANT = "cloudconfig_server__multitenant" + CONFIGPROXY_RPC_PORT = "port_configproxy_rpc" + CONFIGSERVER_RPC_PORT = "port_configserver_rpc" + DEBUG_JVM_STARTUP = "DEBUG_JVM_STARTUP" + DEBUG_STARTUP = "DEBUG_STARTUP" + FILE_DESCRIPTOR_LIMIT = "file_descriptor_limit" + GLIBCXX_FORCE_NEW = "GLIBCXX_FORCE_NEW" + HUGEPAGES_LIST = "HUGEPAGES_LIST" + JAVA_HOME = "JAVA_HOME" + JAVAVM_LD_PRELOAD = "JAVAVM_LD_PRELOAD" + LD_LIBRARY_PATH = "LD_LIBRARY_PATH" + LD_PRELOAD = "LD_PRELOAD" + MADVISE_LIST = "MADVISE_LIST" + MALLOC_ARENA_MAX = "MALLOC_ARENA_MAX" + NO_VESPAMALLOC_LIST = "NO_VESPAMALLOC_LIST" + NUM_PROCESSES_LIMIT = "num_processes_limit" + PATH = "PATH" + PRELOAD = "PRELOAD" + ROOT = "ROOT" + STANDALONE_JDISC_APP_LOCATION = "standalone_jdisc_container__app_location" + STANDALONE_JDISC_DEPLOYMENT_PROFILE = "standalone_jdisc_container__deployment_profile" + STD_THREAD_PREVENT_TRY_CATCH = "STD_THREAD_PREVENT_TRY_CATCH" + TERM = "TERM" + TRACE_JVM_STARTUP = "TRACE_JVM_STARTUP" + TRACE_STARTUP = "TRACE_STARTUP" + VESPA_AFFINITY_CPU_SOCKET = "VESPA_AFFINITY_CPU_SOCKET" + VESPA_ALREADY_SWITCHED_USER_TO = "VESPA_ALREADY_SWITCHED_USER_TO" + VESPA_CLI_API_KEY_FILE = "VESPA_CLI_API_KEY_FILE" + VESPA_CLI_API_KEY = "VESPA_CLI_API_KEY" + VESPA_CLI_CACHE_DIR = "VESPA_CLI_CACHE_DIR" + VESPA_CLI_CLOUD_CI = "VESPA_CLI_CLOUD_CI" + VESPA_CLI_CLOUD_SYSTEM = "VESPA_CLI_CLOUD_SYSTEM" + VESPA_CLI_DATA_PLANE_CERT_FILE = "VESPA_CLI_DATA_PLANE_CERT_FILE" + VESPA_CLI_DATA_PLANE_CERT = "VESPA_CLI_DATA_PLANE_CERT" + VESPA_CLI_DATA_PLANE_KEY_FILE = "VESPA_CLI_DATA_PLANE_KEY_FILE" + VESPA_CLI_DATA_PLANE_KEY = "VESPA_CLI_DATA_PLANE_KEY" + VESPA_CLI_ENDPOINTS = "VESPA_CLI_ENDPOINTS" + VESPA_CLI_HOME = "VESPA_CLI_HOME" + VESPA_CONFIG_ID = "VESPA_CONFIG_ID" + VESPA_CONFIGPROXY_JVMARGS = "VESPA_CONFIGPROXY_JVMARGS" + VESPA_CONFIGSERVER_JVMARGS = "VESPA_CONFIGSERVER_JVMARGS" + VESPA_CONFIGSERVER_MULTITENANT = "VESPA_CONFIGSERVER_MULTITENANT" + VESPA_CONFIGSERVERS = "VESPA_CONFIGSERVERS" + VESPA_CONTAINER_JVMARGS = "VESPA_CONTAINER_JVMARGS" + VESPA_GROUP = "VESPA_GROUP" + VESPA_HOME = "VESPA_HOME" + VESPA_HOSTNAME = "VESPA_HOSTNAME" + VESPA_LOAD_CODE_AS_HUGEPAGES = "VESPA_LOAD_CODE_AS_HUGEPAGES" + VESPA_LOG_CONTROL_DIR = "VESPA_LOG_CONTROL_DIR" + VESPA_LOG_CONTROL_FILE = "VESPA_LOG_CONTROL_FILE" + VESPA_LOG_TARGET = "VESPA_LOG_TARGET" + VESPAMALLOCD_LIST = "VESPAMALLOCD_LIST" + VESPAMALLOCDST_LIST = "VESPAMALLOCDST_LIST" + VESPA_MALLOC_HUGEPAGES = "VESPA_MALLOC_HUGEPAGES" + VESPAMALLOC_LIST = "VESPAMALLOCD_LIST" + VESPA_MALLOC_MADVISE_LIMIT = "VESPA_MALLOC_MADVISE_LIMIT" + VESPA_NO_NUMACTL = "VESPA_NO_NUMACTL" + VESPA_ONLY_IP_V6_NETWORKING = "VESPA_ONLY_IP_V6_NETWORKING" + VESPA_PORT_BASE = "VESPA_PORT_BASE" + VESPA_SERVICE_NAME = "VESPA_SERVICE_NAME" + VESPA_TIMER_HZ = "VESPA_TIMER_HZ" + VESPA_TLS_CA_CERT = "VESPA_TLS_CA_CERT" + VESPA_TLS_CERT = "VESPA_TLS_CERT" + VESPA_TLS_CONFIG_FILE = "VESPA_TLS_CONFIG_FILE" + VESPA_TLS_ENABLED = "VESPA_TLS_ENABLED" + VESPA_TLS_HOSTNAME_VALIDATION_DISABLED = "VESPA_TLS_HOSTNAME_VALIDATION_DISABLED" + VESPA_TLS_INSECURE_MIXED_MODE = "VESPA_TLS_INSECURE_MIXED_MODE" + VESPA_TLS_PRIVATE_KEY = "VESPA_TLS_PRIVATE_KEY" + VESPA_USE_HUGEPAGES_LIST = "VESPA_USE_HUGEPAGES_LIST" + VESPA_USE_HUGEPAGES = "VESPA_USE_HUGEPAGES" + VESPA_USE_MADVISE_LIST = "VESPA_USE_MADVISE_LIST" + VESPA_USE_NO_VESPAMALLOC = "VESPA_USE_NO_VESPAMALLOC" + VESPA_USER = "VESPA_USER" + VESPA_USE_VALGRIND = "VESPA_USE_VALGRIND" + VESPA_USE_VESPAMALLOC_DST = "VESPA_USE_VESPAMALLOC_DST" + VESPA_USE_VESPAMALLOC_D = "VESPA_USE_VESPAMALLOC_D" + VESPA_USE_VESPAMALLOC = "VESPA_USE_VESPAMALLOC" + VESPA_VALGRIND_OPT = "VESPA_VALGRIND_OPT" + VESPA_WEB_SERVICE_PORT = "VESPA_WEB_SERVICE_PORT" +) diff --git a/client/go/internal/admin/jvm/application_container.go b/client/go/internal/admin/jvm/application_container.go new file mode 100644 index 00000000000..59e87d22ead --- /dev/null +++ b/client/go/internal/admin/jvm/application_container.go @@ -0,0 +1,173 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/prog" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const ( + JAR_FOR_APPLICATION_CONTAINER = "container-disc-jar-with-dependencies.jar" +) + +type ApplicationContainer struct { + containerBase +} + +func (a *ApplicationContainer) ArgForMain() string { + dir := defaults.UnderVespaHome("lib/jars") + return fmt.Sprintf("file:%s/%s", dir, JAR_FOR_APPLICATION_CONTAINER) +} + +func (a *ApplicationContainer) Discriminator() string { + cfgId := a.ConfigId() + if cfgId != "" { + trace.Trace("Discriminator: using md5 of", cfgId) + return util.Md5Hex(cfgId + "\n") + } + svcName := a.ServiceName() + if svcName != "" { + trace.Trace("Discriminator: using", svcName) + return svcName + } + pid := os.Getpid() + trace.Trace("Discriminator: using md5 of", pid) + return util.Md5Hex(fmt.Sprintf("%d", pid)) +} + +func (a *ApplicationContainer) addJdiscProperties() { + cfgId := a.ConfigId() + opts := a.jvmOpts + opts.AddCommonJdiscProperties() + containerParentDir := defaults.UnderVespaHome("var/jdisc_container") + containerHomeDir := fmt.Sprintf("%s/%s", containerParentDir, a.Discriminator()) + bCacheDir := fmt.Sprintf("%s/%s", containerHomeDir, "bundlecache") + propsFile := fmt.Sprintf("%s/%s.properties", containerHomeDir, "jdisc") + opts.fixSpec.FixDir(containerHomeDir) + opts.fixSpec.FixDir(bCacheDir) + a.propsFile = propsFile + opts.AddOption("-Djdisc.config.file=" + propsFile) + opts.AddOption("-Djdisc.cache.path=" + bCacheDir) + opts.AddOption("-Djdisc.logger.tag=" + cfgId) +} + +func validPercentage(val int) bool { + return val > 0 && val < 100 +} + +func (a *ApplicationContainer) configureMemory(qc *QrStartConfig) { + jvm_heapsize := qc.Jvm.Heapsize // Heap size (in megabytes) for the Java VM + jvm_minHeapsize := qc.Jvm.MinHeapsize // Min heapsize (in megabytes) for the Java VM + jvm_stacksize := qc.Jvm.Stacksize // Stack size (in kilobytes) + jvm_compressedClassSpaceSize := qc.Jvm.CompressedClassSpaceSize // CompressedOOps size in megabytes + jvm_baseMaxDirectMemorySize := qc.Jvm.BaseMaxDirectMemorySize // Base value of maximum direct memory size (in megabytes) + jvm_directMemorySizeCache := qc.Jvm.DirectMemorySizeCache // Amount of direct memory used for caching. (in megabytes) + jvm_heapSizeAsPercentageOfPhysicalMemory := qc.Jvm.HeapSizeAsPercentageOfPhysicalMemory // Heap size as percentage of available RAM, overrides value above. + + if jvm_heapsize <= 0 { + jvm_heapsize = 1536 + trace.Trace("using hardcoded value for jvm_heapsize:", jvm_heapsize) + } + if jvm_minHeapsize <= 0 { + jvm_minHeapsize = jvm_heapsize + } + available := getAvailableMemory() + if validPercentage(jvm_heapSizeAsPercentageOfPhysicalMemory) && available.ToMB() > 500 { + available = adjustAvailableMemory(available) + jvm_heapsize = available.ToMB() * jvm_heapSizeAsPercentageOfPhysicalMemory / 100 + jvm_minHeapsize = jvm_heapsize + } + if jvm_minHeapsize > jvm_heapsize { + trace.Warning(fmt.Sprintf( + "Misconfigured heap size, jvm_minHeapsize(%d) is larger than jvm_heapsize(%d). It has been capped.", + jvm_minHeapsize, jvm_heapsize)) + jvm_minHeapsize = jvm_heapsize + } + if jvm_stacksize <= 0 { + jvm_stacksize = 512 + } + if jvm_baseMaxDirectMemorySize <= 0 { + jvm_baseMaxDirectMemorySize = 75 + } + if jvm_compressedClassSpaceSize < 0 { + jvm_compressedClassSpaceSize = 32 + } + if jvm_directMemorySizeCache < 0 { + jvm_directMemorySizeCache = 0 + } + maxDirectMemorySize := jvm_baseMaxDirectMemorySize + (jvm_heapsize / 8) + jvm_directMemorySizeCache + opts := a.jvmOpts + opts.AddOption(fmt.Sprintf("-Xms%dm", jvm_minHeapsize)) + opts.AddOption(fmt.Sprintf("-Xmx%dm", jvm_heapsize)) + opts.AddOption(fmt.Sprintf("-XX:ThreadStackSize=%d", jvm_stacksize)) + opts.AddOption(fmt.Sprintf("-XX:MaxDirectMemorySize=%dm", maxDirectMemorySize)) + opts.MaybeAddHugepages(MegaBytesOfMemory(jvm_heapsize)) + if jvm_compressedClassSpaceSize > 0 { + opts.AddOption(fmt.Sprintf("-XX:CompressedClassSpaceSize=%dm", jvm_compressedClassSpaceSize)) + } +} + +func (a *ApplicationContainer) configureGC(qc *QrStartConfig) { + if extra := qc.Jvm.Gcopts; extra != "" { + a.JvmOptions().AddJvmArgsFromString(extra) + } + if qc.Jvm.Verbosegc { + a.JvmOptions().AddOption("-Xlog:gc") + } +} + +func (a *ApplicationContainer) configureClasspath(qc *QrStartConfig) { + opts := a.JvmOptions() + if cp := qc.Jdisc.ClasspathExtra; cp != "" { + opts.classPath = append(opts.classPath, cp) + } +} + +func (a *ApplicationContainer) configureCPU(qc *QrStartConfig) { + cnt := qc.Jvm.AvailableProcessors + if cnt > 0 { + trace.Trace("CpuCount: using", cnt, "from qr-start config") + } + a.JvmOptions().ConfigureCpuCount(cnt) +} + +func (a *ApplicationContainer) configureOptions() { + opts := a.JvmOptions() + opts.AddOption("-Dconfig.id=" + a.ConfigId()) + if env := os.Getenv(envvars.VESPA_CONTAINER_JVMARGS); env != "" { + opts.AddJvmArgsFromString(env) + } + qrStartCfg := a.getQrStartCfg() + opts.AddOption("-Djdisc.export.packages=" + qrStartCfg.Jdisc.ExportPackages) + opts.AddCommonXX() + opts.AddCommonOpens() + opts.AddCommonJdkProperties() + a.configureCPU(qrStartCfg) + a.configureMemory(qrStartCfg) + a.configureGC(qrStartCfg) + a.configureClasspath(qrStartCfg) + a.addJdiscProperties() + svcName := a.ServiceName() + if svcName == "container" || svcName == "container-clustercontroller" { + RemoveStaleZkLocks(a) + logsDir := defaults.UnderVespaHome("logs/vespa") + zkLogFile := fmt.Sprintf("%s/zookeeper.%s", logsDir, svcName) + opts.AddOption("-Dzookeeper_log_file_prefix=" + zkLogFile) + } +} + +func (c *ApplicationContainer) exportExtraEnv(ps *prog.Spec) { + if c.ConfigId() != "" { + ps.Setenv(envvars.VESPA_CONFIG_ID, c.ConfigId()) + } else { + util.JustExitMsg("application container requires a config id") + } +} diff --git a/client/go/internal/admin/jvm/configproxy_jvm.go b/client/go/internal/admin/jvm/configproxy_jvm.go new file mode 100644 index 00000000000..1c274d8b889 --- /dev/null +++ b/client/go/internal/admin/jvm/configproxy_jvm.go @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/prog" +) + +const ( + PROXY_JAR_FILE = "lib/jars/config-proxy-jar-with-dependencies.jar" + PROXY_MAIN_CLASS = "com.yahoo.vespa.config.proxy.ProxyServer" + PROXY_PORT_ARG = "19090" +) + +type ConfigProxyJvm struct { + containerBase +} + +func (cpc *ConfigProxyJvm) ArgForMain() string { + return PROXY_PORT_ARG +} + +func (cpc *ConfigProxyJvm) ConfigId() string { + return "" +} + +func (cpc *ConfigProxyJvm) ConfigureOptions(configsources []string, userargs string) { + opts := cpc.jvmOpts + opts.jarWithDeps = PROXY_JAR_FILE + opts.mainClass = PROXY_MAIN_CLASS + opts.AddOption("-XX:+ExitOnOutOfMemoryError") + opts.AddOption("-XX:+PreserveFramePointer") + opts.AddOption("-XX:CompressedClassSpaceSize=32m") + opts.AddOption("-XX:MaxDirectMemorySize=32m") + opts.AddOption("-XX:ThreadStackSize=448") + opts.AddOption("-XX:MaxJavaStackTraceDepth=1000") + opts.AddOption("-XX:-OmitStackTraceInFastThrow") + opts.AddOption("-XX:ActiveProcessorCount=2") + opts.AddOption("-Dproxyconfigsources=" + strings.Join(configsources, ",")) + opts.AddOption("-Djava.io.tmpdir=${VESPA_HOME}/var/tmp") + if userargs != "" { + opts.AddJvmArgsFromString(userargs) + } + minFallback := MegaBytesOfMemory(32) + maxFallback := MegaBytesOfMemory(128) + opts.AddDefaultHeapSizeArgs(minFallback, maxFallback) +} + +func (cpc *ConfigProxyJvm) exportExtraEnv(ps *prog.Spec) { +} + +func NewConfigProxyJvm(serviceName string) *ConfigProxyJvm { + var cpc ConfigProxyJvm + cpc.serviceName = serviceName + cpc.jvmOpts = NewOptions(&cpc) + return &cpc +} diff --git a/client/go/internal/admin/jvm/container.go b/client/go/internal/admin/jvm/container.go new file mode 100644 index 00000000000..d7bdeae43b6 --- /dev/null +++ b/client/go/internal/admin/jvm/container.go @@ -0,0 +1,75 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "fmt" + "sort" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/prog" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +type Container interface { + ServiceName() string + ConfigId() string + ArgForMain() string + JvmOptions() *Options + Exec() + exportExtraEnv(ps *prog.Spec) +} + +type containerBase struct { + configId string + serviceName string + jvmOpts *Options + propsFile string +} + +func (cb *containerBase) ServiceName() string { + return cb.serviceName +} + +func (cb *containerBase) JvmOptions() *Options { + return cb.jvmOpts +} + +func (cb *containerBase) ConfigId() string { + return cb.configId +} + +func keysOfMap(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k, _ := range m { + keys = append(keys, k) + } + return keys +} + +func readableEnv(env map[string]string) string { + keys := keysOfMap(env) + sort.Strings(keys) + var buf strings.Builder + for _, k := range keys { + fmt.Fprintf(&buf, " %s=%s", k, env[k]) + } + return buf.String() +} + +func (cb *containerBase) Exec() { + argv := util.ArrayListOf(cb.JvmOptions().Args()) + argv.Insert(0, "java") + p := prog.NewSpec(argv) + p.ConfigureNumaCtl() + cb.JvmOptions().exportEnvSettings(p) + if cb.propsFile != "" { + writeEnvAsProperties(p.EffectiveEnv(), cb.propsFile) + } + trace.Info("starting container; env:", readableEnv(p.Env)) + trace.Info("starting container; exec:", argv) + err := p.Run() + util.JustExitWith(err) +} diff --git a/client/go/internal/admin/jvm/env.go b/client/go/internal/admin/jvm/env.go new file mode 100644 index 00000000000..7b1ce97a40a --- /dev/null +++ b/client/go/internal/admin/jvm/env.go @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/prog" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func (opts *Options) exportEnvSettings(ps *prog.Spec) { + c := opts.container + vespaHome := defaults.VespaHome() + lvd := fmt.Sprintf("%s/logs/vespa", vespaHome) + vlt := fmt.Sprintf("file:%s/vespa.log", lvd) + lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome) + lcf := fmt.Sprintf("%s/%s.logcontrol", lcd, c.ServiceName()) + dlp := fmt.Sprintf("%s/lib64:/opt/vespa-deps/lib64", vespaHome) + opts.fixSpec.FixDir(lvd) + opts.fixSpec.FixDir(lcd) + ps.Setenv(envvars.VESPA_LOG_TARGET, vlt) + ps.Setenv(envvars.VESPA_LOG_CONTROL_DIR, lcd) + ps.Setenv(envvars.VESPA_LOG_CONTROL_FILE, lcf) + ps.Setenv(envvars.VESPA_SERVICE_NAME, c.ServiceName()) + ps.Setenv(envvars.LD_LIBRARY_PATH, dlp) + ps.Setenv(envvars.MALLOC_ARENA_MAX, "1") + if preload := ps.Getenv(envvars.PRELOAD); preload != "" { + ps.Setenv(envvars.JAVAVM_LD_PRELOAD, preload) + ps.Setenv(envvars.LD_PRELOAD, preload) + } + util.OptionallyReduceTimerFrequency() + c.exportExtraEnv(ps) +} diff --git a/client/go/internal/admin/jvm/jdisc_options.go b/client/go/internal/admin/jvm/jdisc_options.go new file mode 100644 index 00000000000..3e6c5b6003d --- /dev/null +++ b/client/go/internal/admin/jvm/jdisc_options.go @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" +) + +const ( + DEFAULT_MAIN_CLASS = "com.yahoo.jdisc.core.StandaloneMain" + DEFAULT_JAR_WITH_DEPS = "lib/jars/jdisc_core-jar-with-dependencies.jar" +) + +func (opts *Options) AddCommonJdiscProperties() { + logCtlDir := defaults.UnderVespaHome("var/db/vespa/logcontrol") + opts.fixSpec.FixDir(logCtlDir) + opts.AddOption("-Djdisc.bundle.path=" + defaults.UnderVespaHome("lib/jars")) + opts.AddOption("-Djdisc.logger.enabled=false") + opts.AddOption("-Djdisc.logger.level=WARNING") + opts.AddOption("-Dvespa.log.control.dir=" + logCtlDir) +} diff --git a/client/go/internal/admin/jvm/jdk_properties.go b/client/go/internal/admin/jvm/jdk_properties.go new file mode 100644 index 00000000000..120771234de --- /dev/null +++ b/client/go/internal/admin/jvm/jdk_properties.go @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" +) + +func (opts *Options) AddCommonJdkProperties() { + tmpDir := defaults.UnderVespaHome("var/tmp") + libDir := defaults.UnderVespaHome("lib64") + secOvr := defaults.UnderVespaHome("conf/vespa/java.security.override") + opts.fixSpec.FixDir(tmpDir) + opts.AddOption("-Djava.io.tmpdir=" + tmpDir) + opts.AddOption("-Djava.library.path=" + libDir + ":/opt/vespa-deps/lib64") + opts.AddOption("-Djava.security.properties=" + secOvr) + opts.AddOption("-Djava.awt.headless=true") + opts.AddOption("-Dsun.rmi.dgc.client.gcInterval=3600000") + opts.AddOption("-Dsun.net.client.defaultConnectTimeout=5000") + opts.AddOption("-Dsun.net.client.defaultReadTimeout=60000") + opts.AddOption("-Djavax.net.ssl.keyStoreType=JKS") + opts.AddOption("-Djdk.tls.rejectClientInitiatedRenegotiation=true") + opts.AddOption("-Dfile.encoding=UTF-8") + opts.AddOption("-Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger") + if env := os.Getenv(envvars.VESPA_ONLY_IP_V6_NETWORKING); env == "true" { + opts.AddOption("-Djava.net.preferIPv6Addresses=true") + } +} diff --git a/client/go/internal/admin/jvm/mem_avail.go b/client/go/internal/admin/jvm/mem_avail.go new file mode 100644 index 00000000000..11203b9aecf --- /dev/null +++ b/client/go/internal/admin/jvm/mem_avail.go @@ -0,0 +1,142 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "fmt" + "os" + "strconv" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func parseFree(txt string) AmountOfMemory { + f := strings.Fields(txt) + for idx, field := range f { + if field == "Mem:" && idx+1 < len(f) { + res, err := strconv.Atoi(f[idx+1]) + if err == nil { + return MegaBytesOfMemory(res) + } else { + trace.Warning(err) + } + } + } + return BytesOfMemory(0) +} + +func parentDir(dir string) string { + lastSlash := 0 + for idx, ch := range dir { + if ch == '/' { + lastSlash = idx + } + } + return dir[:lastSlash] +} + +func readLineFrom(filename string) (string, error) { + content, err := os.ReadFile(filename) + s := string(content) + if err != nil { + return s, err + } + s = strings.TrimSuffix(s, "\n") + if strings.Contains(s, "\n") { + return s, fmt.Errorf("unexpected multiple lines in file %s", filename) + } + return s, nil +} + +func vespa_cg2get(limitname string) (output string, err error) { + return vespa_cg2get_impl("", limitname) +} +func vespa_cg2get_impl(rootdir, limitname string) (output string, err error) { + _, err = os.Stat(rootdir + "/sys/fs/cgroup/cgroup.controllers") + if err != nil { + trace.Trace("no cgroups:", err) + return + } + cgroup_content, err := readLineFrom(rootdir + "/proc/self/cgroup") + if err != nil { + trace.Trace("no cgroup for self:", err) + return + } + min_value := "max" + path := rootdir + "/sys/fs/cgroup" + slice := strings.TrimPrefix(cgroup_content, "0::") + dirNames := strings.Split(slice, "/") + for _, dirName := range dirNames { + path = path + dirName + "/" + value, err := readLineFrom(path + limitname) + trace.Debug("read from", path+limitname, "=>", value) + if err == nil { + if value == "max" { + // nop + } else if min_value == "max" { + min_value = value + } else if len(value) < len(min_value) { + min_value = value + } else if len(value) == len(min_value) && value < min_value { + min_value = value + } + } + } + trace.Trace("min_value of", limitname, "for cgroups v2:", min_value) + return min_value, nil +} + +func getAvailableMemory() AmountOfMemory { + result := BytesOfMemory(0) + backticks := util.BackTicksWithStderr + freeOutput, err := backticks.Run("free", "-m") + if err == nil { + result = parseFree(freeOutput) + trace.Trace("run 'free' ok, result:", result) + } else { + trace.Trace("run 'free' failed:", err) + } + available_cgroup := KiloBytesOfMemory(1 << 31) + cggetOutput, err := backticks.Run("cgget", "-nv", "-r", "memory.limit_in_bytes", "/") + if err != nil { + cggetOutput, err = vespa_cg2get("memory.max") + } + cggetOutput = strings.TrimSpace(cggetOutput) + if err != nil { + trace.Debug("run 'cgget' failed:", err, "=>", cggetOutput) + } + if err == nil && cggetOutput != "max" { + numBytes, err := strconv.ParseInt(cggetOutput, 10, 64) + if err == nil && numBytes > (1<<28) { + available_cgroup = BytesOfMemory(numBytes) + } else { + trace.Warning("unexpected 'cgget' output:", cggetOutput) + } + } + if result.ToKB() == 0 || result.ToKB() > available_cgroup.ToKB() { + result = available_cgroup + } + trace.Trace("getAvailableMemory returns:", result) + return result +} + +func getTransparentHugepageSize() AmountOfMemory { + const fn = "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" + thp_size := MegaBytesOfMemory(2) + line, err := readLineFrom(fn) + if err == nil { + number, err := strconv.ParseInt(line, 10, 64) + if err == nil { + thp_size = BytesOfMemory(number) + trace.Trace("thp_size", line, "=>", thp_size) + } else { + trace.Trace("no thp_size:", err) + } + } else { + trace.Trace("no thp_size:", err) + } + return thp_size +} diff --git a/client/go/internal/admin/jvm/mem_avail_test.go b/client/go/internal/admin/jvm/mem_avail_test.go new file mode 100644 index 00000000000..ea92fbfa858 --- /dev/null +++ b/client/go/internal/admin/jvm/mem_avail_test.go @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package jvm + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +func TestCg2Get(t *testing.T) { + trace.AdjustVerbosity(0) + const MM = "memory.max" + res, err := vespa_cg2get(MM) + + res, err = vespa_cg2get_impl("mock-cg2/a", MM) + assert.Nil(t, err) + assert.Equal(t, "123", res) + + res, err = vespa_cg2get_impl("mock-cg2/b", MM) + assert.Nil(t, err) + assert.Equal(t, "67430985728", res) + + res, err = vespa_cg2get_impl("mock-cg2/c", MM) + assert.Nil(t, err) + assert.Equal(t, "9663676416", res) +} + +func TestParseFree(t *testing.T) { + res := parseFree(` + total used free shared buff/cache available +Mem: 19986 656 3157 218 16172 18832 +Swap: 2047 320 1727 +`) + assert.Equal(t, MegaBytesOfMemory(19986), res) +} + +func TestGetAvail(t *testing.T) { + trace.AdjustVerbosity(0) + available := getAvailableMemory() + assert.True(t, available.ToMB() >= 0) +} diff --git a/client/go/internal/admin/jvm/mem_options.go b/client/go/internal/admin/jvm/mem_options.go new file mode 100644 index 00000000000..f58bb141587 --- /dev/null +++ b/client/go/internal/admin/jvm/mem_options.go @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "fmt" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +func (opts *Options) getOrSetHeapSize(prefix string, heapSize AmountOfMemory) AmountOfMemory { + var missing bool = true + for _, x := range opts.jvmArgs { + if strings.HasPrefix(x, prefix) { + val, err := ParseJvmMemorySpec(strings.TrimPrefix(x, prefix)) + if err == nil { + missing = false + heapSize = val + } + } + } + if missing { + opts.AppendOption(fmt.Sprintf("%s%s", prefix, heapSize.AsJvmSpec())) + } + return heapSize +} + +func (opts *Options) CurMinHeapSize(fallback AmountOfMemory) AmountOfMemory { + return opts.getOrSetHeapSize("-Xms", fallback) +} + +func (opts *Options) CurMaxHeapSize(fallback AmountOfMemory) AmountOfMemory { + return opts.getOrSetHeapSize("-Xmx", fallback) +} + +func (opts *Options) AddDefaultHeapSizeArgs(minHeapSize, maxHeapSize AmountOfMemory) { + trace.Trace("AddDefaultHeapSizeArgs", minHeapSize, "/", maxHeapSize) + minHeapSize = opts.CurMinHeapSize(minHeapSize) + maxHeapSize = opts.CurMaxHeapSize(maxHeapSize) + opts.MaybeAddHugepages(maxHeapSize) +} + +func (opts *Options) MaybeAddHugepages(heapSize AmountOfMemory) { + thpSize := getTransparentHugepageSize() + if thpSize.numBytes*2 < heapSize.numBytes { + trace.Trace("add UseTransparentHugePages, 2 * thpSize", thpSize, " < maxHeap", heapSize) + opts.AddOption("-XX:+UseTransparentHugePages") + } else { + trace.Trace("no UseTransparentHugePages, 2 * thpSize", thpSize, " >= maxHeap", heapSize) + } +} + +func adjustAvailableMemory(measured AmountOfMemory) AmountOfMemory { + reserved := 1024 // MB + need_min := 64 // MB + available := measured.ToMB() + if available > need_min+2*reserved { + return MegaBytesOfMemory(available - reserved) + } + if available > need_min { + adjusted := (available + need_min) / 2 + return MegaBytesOfMemory(adjusted) + } + return MegaBytesOfMemory(need_min) +} diff --git a/client/go/internal/admin/jvm/mem_options_test.go b/client/go/internal/admin/jvm/mem_options_test.go new file mode 100644 index 00000000000..c15143d4758 --- /dev/null +++ b/client/go/internal/admin/jvm/mem_options_test.go @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package jvm + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAdjustment(t *testing.T) { + lastAdj := 64 + for i := 0; i < 4096; i++ { + adj := adjustAvailableMemory(MegaBytesOfMemory(i)).ToMB() + assert.True(t, int(adj) >= lastAdj) + lastAdj = int(adj) + } + adj := adjustAvailableMemory(MegaBytesOfMemory(31024)).ToMB() + assert.Equal(t, 30000, int(adj)) +} diff --git a/client/go/internal/admin/jvm/memory.go b/client/go/internal/admin/jvm/memory.go new file mode 100644 index 00000000000..8caa1a3be22 --- /dev/null +++ b/client/go/internal/admin/jvm/memory.go @@ -0,0 +1,89 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "fmt" +) + +const ( + _ = 1 << (10 * iota) + PowerOfTwo10 + PowerOfTwo20 + PowerOfTwo30 +) + +type AmountOfMemory struct { + numBytes int64 +} + +func BytesOfMemory(v int64) AmountOfMemory { + return AmountOfMemory{numBytes: v} +} +func KiloBytesOfMemory(v int64) AmountOfMemory { + return BytesOfMemory(v * PowerOfTwo10) +} +func MegaBytesOfMemory(v int) AmountOfMemory { + return BytesOfMemory(int64(v) * PowerOfTwo20) +} +func GigaBytesOfMemory(v int) AmountOfMemory { + return BytesOfMemory(int64(v) * PowerOfTwo30) +} + +func (v AmountOfMemory) ToBytes() int64 { + return v.numBytes +} +func (v AmountOfMemory) ToKB() int64 { + return v.numBytes / PowerOfTwo10 +} +func (v AmountOfMemory) ToMB() int { + return int(v.numBytes / PowerOfTwo20) +} +func (v AmountOfMemory) ToGB() int { + return int(v.numBytes / PowerOfTwo30) +} +func (v AmountOfMemory) AsJvmSpec() string { + val := v.ToKB() + suffix := "k" + if val%PowerOfTwo10 == 0 { + val = val / PowerOfTwo10 + suffix = "m" + if val%PowerOfTwo10 == 0 { + val = val / PowerOfTwo10 + suffix = "g" + } + } + return fmt.Sprintf("%d%s", val, suffix) +} +func (v AmountOfMemory) String() string { + val := v.numBytes + idx := 0 + suffix := [9]string{"bytes", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"} + for val > 0 && (val%PowerOfTwo10 == 0) { + val = val / PowerOfTwo10 + idx++ + } + return fmt.Sprintf("{%d %s}", val, suffix[idx]) +} + +func ParseJvmMemorySpec(spec string) (result AmountOfMemory, err error) { + result = BytesOfMemory(0) + var n int + var val int64 + var suffix rune + n, err = fmt.Sscanf(spec, "%d%c", &val, &suffix) + if n == 2 && err == nil { + switch suffix { + case 'k': + result = KiloBytesOfMemory(val) + case 'm': + result = MegaBytesOfMemory(int(val)) + case 'g': + result = GigaBytesOfMemory(int(val)) + default: + err = fmt.Errorf("Unknown suffix in JVM memory spec '%s'", spec) + } + } + return +} diff --git a/client/go/internal/admin/jvm/memory_test.go b/client/go/internal/admin/jvm/memory_test.go new file mode 100644 index 00000000000..c898606a2db --- /dev/null +++ b/client/go/internal/admin/jvm/memory_test.go @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package jvm + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConversion(t *testing.T) { + v1 := GigaBytesOfMemory(17) + v2 := MegaBytesOfMemory(17 * 1024) + v3 := KiloBytesOfMemory(17 * 1024 * 1024) + var numBytes int64 = 17 * 1024 * 1024 * 1024 + assert.Equal(t, v1, v2) + assert.Equal(t, v1, v3) + assert.Equal(t, v2, v1) + assert.Equal(t, v2, v3) + assert.Equal(t, v3, v1) + assert.Equal(t, v3, v2) + assert.Equal(t, numBytes, v1.ToBytes()) + assert.Equal(t, numBytes, v2.ToBytes()) + assert.Equal(t, numBytes, v3.ToBytes()) + assert.Equal(t, "17g", v1.AsJvmSpec()) + assert.Equal(t, "17g", v2.AsJvmSpec()) + assert.Equal(t, "17g", v3.AsJvmSpec()) + + v1 = GigaBytesOfMemory(17) + v2 = MegaBytesOfMemory(17 * 1000) + v3 = KiloBytesOfMemory(17 * 1000 * 1000) + assert.Equal(t, "17g", v1.AsJvmSpec()) + assert.Equal(t, "17000m", v2.AsJvmSpec()) + assert.Equal(t, "17000000k", v3.AsJvmSpec()) + assert.Equal(t, "{17 GiB}", v1.String()) + assert.Equal(t, "{17000 MiB}", v2.String()) + assert.Equal(t, "{17000000 KiB}", v3.String()) + + var result AmountOfMemory + var err error + result, err = ParseJvmMemorySpec("17g") + assert.Nil(t, err) + assert.Equal(t, v1, result) + result, err = ParseJvmMemorySpec("17000m") + assert.Nil(t, err) + assert.Equal(t, v2, result) + result, err = ParseJvmMemorySpec("17000000k") + assert.Nil(t, err) + assert.Equal(t, v3, result) +} diff --git a/client/go/internal/admin/jvm/mock-cg2/a/proc/self/cgroup b/client/go/internal/admin/jvm/mock-cg2/a/proc/self/cgroup new file mode 100644 index 00000000000..2cc2de315da --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/a/proc/self/cgroup @@ -0,0 +1 @@ +0::/system.slice/sshd.service diff --git a/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/cgroup.controllers b/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/cgroup.controllers new file mode 100644 index 00000000000..acb4d263e65 --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/cgroup.controllers @@ -0,0 +1 @@ +cpuset cpu io memory hugetlb pids rdma diff --git a/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/memory.max b/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/memory.max new file mode 100644 index 00000000000..355295a05af --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/memory.max @@ -0,0 +1 @@ +max diff --git a/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/system.slice/memory.max b/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/system.slice/memory.max new file mode 100644 index 00000000000..190a18037c6 --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/system.slice/memory.max @@ -0,0 +1 @@ +123 diff --git a/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/system.slice/sshd.service/memory.max b/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/system.slice/sshd.service/memory.max new file mode 100644 index 00000000000..355295a05af --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/a/sys/fs/cgroup/system.slice/sshd.service/memory.max @@ -0,0 +1 @@ +max diff --git a/client/go/internal/admin/jvm/mock-cg2/b/proc/self/cgroup b/client/go/internal/admin/jvm/mock-cg2/b/proc/self/cgroup new file mode 100644 index 00000000000..8dc7ddcf52f --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/b/proc/self/cgroup @@ -0,0 +1 @@ +0::/init.scope diff --git a/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/cgroup.controllers b/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/cgroup.controllers new file mode 100644 index 00000000000..acb4d263e65 --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/cgroup.controllers @@ -0,0 +1 @@ +cpuset cpu io memory hugetlb pids rdma diff --git a/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/init.scope/memory.max b/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/init.scope/memory.max new file mode 100644 index 00000000000..355295a05af --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/init.scope/memory.max @@ -0,0 +1 @@ +max diff --git a/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/init.scope/sshd.service/memory.max b/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/init.scope/sshd.service/memory.max new file mode 100644 index 00000000000..355295a05af --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/init.scope/sshd.service/memory.max @@ -0,0 +1 @@ +max diff --git a/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/memory.max b/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/memory.max new file mode 100644 index 00000000000..ca65b2abde1 --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/b/sys/fs/cgroup/memory.max @@ -0,0 +1 @@ +67430985728 diff --git a/client/go/internal/admin/jvm/mock-cg2/c/proc/self/cgroup b/client/go/internal/admin/jvm/mock-cg2/c/proc/self/cgroup new file mode 100644 index 00000000000..c76235307d0 --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/c/proc/self/cgroup @@ -0,0 +1 @@ +0::/system.slice/vespa.service diff --git a/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/cgroup.controllers b/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/cgroup.controllers new file mode 100644 index 00000000000..acb4d263e65 --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/cgroup.controllers @@ -0,0 +1 @@ +cpuset cpu io memory hugetlb pids rdma diff --git a/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/memory.max b/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/memory.max new file mode 100644 index 00000000000..ca65b2abde1 --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/memory.max @@ -0,0 +1 @@ +67430985728 diff --git a/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/system.slice/memory.max b/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/system.slice/memory.max new file mode 100644 index 00000000000..355295a05af --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/system.slice/memory.max @@ -0,0 +1 @@ +max diff --git a/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/system.slice/vespa.service/memory.max b/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/system.slice/vespa.service/memory.max new file mode 100644 index 00000000000..8061635036a --- /dev/null +++ b/client/go/internal/admin/jvm/mock-cg2/c/sys/fs/cgroup/system.slice/vespa.service/memory.max @@ -0,0 +1 @@ +9663676416 diff --git a/client/go/internal/admin/jvm/opens_options.go b/client/go/internal/admin/jvm/opens_options.go new file mode 100644 index 00000000000..bfc4e519787 --- /dev/null +++ b/client/go/internal/admin/jvm/opens_options.go @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "fmt" +) + +func (opts *Options) AddOpens(module, pkg string) { + opt := fmt.Sprintf("--add-opens=%s/%s=ALL-UNNAMED", module, pkg) + opts.AddOption(opt) +} + +func (opts *Options) addOpensJB(pkg string) { + opts.AddOpens("java.base", pkg) +} + +func (opts *Options) AddCommonOpens() { + opts.addOpensJB("java.io") + opts.addOpensJB("java.lang") + opts.addOpensJB("java.net") + opts.addOpensJB("java.nio") + opts.addOpensJB("jdk.internal.loader") + opts.addOpensJB("sun.security.ssl") +} diff --git a/client/go/internal/admin/jvm/options.go b/client/go/internal/admin/jvm/options.go new file mode 100644 index 00000000000..f509b4d44d1 --- /dev/null +++ b/client/go/internal/admin/jvm/options.go @@ -0,0 +1,98 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "fmt" + "strconv" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +type Options struct { + container Container + classPath []string + jvmArgs util.ArrayList[string] + mainClass string + jarWithDeps string + fixSpec util.FixSpec +} + +func NewOptions(c Container) *Options { + vespaUid, vespaGid := vespa.FindVespaUidAndGid() + fixSpec := util.FixSpec{ + UserId: vespaUid, + GroupId: vespaGid, + DirMode: 0755, + FileMode: 0644, + } + return &Options{ + container: c, + classPath: make([]string, 0, 10), + jvmArgs: make([]string, 0, 100), + jarWithDeps: DEFAULT_JAR_WITH_DEPS, + mainClass: DEFAULT_MAIN_CLASS, + fixSpec: fixSpec, + } +} + +func (opts *Options) AddOption(arg string) { + if opts.jvmArgs.Contains(arg) { + return + } + opts.AppendOption(arg) +} + +func (opts *Options) AppendOption(arg string) { + trace.Trace("append JVM option:", arg) + opts.jvmArgs = append(opts.jvmArgs, arg) +} + +func (opts *Options) ClassPath() string { + cp := defaults.UnderVespaHome(opts.jarWithDeps) + for _, x := range opts.classPath { + cp = fmt.Sprintf("%s:%s", cp, x) + } + trace.Trace("computed classpath:", cp) + return cp +} + +func (opts *Options) Args() []string { + args := opts.jvmArgs + args = append(args, "-cp") + args = append(args, opts.ClassPath()) + args = append(args, opts.mainClass) + args = append(args, opts.container.ArgForMain()) + return args +} + +func (opts *Options) AddJvmArgsFromString(args string) { + for _, x := range strings.Fields(args) { + opts.AppendOption(x) + } +} + +func (opts *Options) ConfigureCpuCount(cnt int) { + if cnt <= 0 { + out, err := util.BackTicksForwardStderr.Run("nproc", "--all") + if err != nil { + trace.Trace("failed nproc:", err) + } else { + cnt, err = strconv.Atoi(strings.TrimSpace(out)) + if err != nil { + trace.Trace("bad nproc output:", strings.TrimSpace(out)) + cnt = 0 + } else { + trace.Trace("CpuCount: using", cnt, "from nproc --all") + } + } + } + if cnt > 0 { + opts.AddOption(fmt.Sprintf("-XX:ActiveProcessorCount=%d", cnt)) + } +} diff --git a/client/go/internal/admin/jvm/options_test.go b/client/go/internal/admin/jvm/options_test.go new file mode 100644 index 00000000000..0f781756dde --- /dev/null +++ b/client/go/internal/admin/jvm/options_test.go @@ -0,0 +1,88 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package jvm + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/admin/prog" +) + +type dummyContainer struct{ containerBase } + +func (*dummyContainer) ArgForMain() string { return "arg-for-main" } +func (*dummyContainer) exportExtraEnv(ps *prog.Spec) {} +func newDummyContainer() Container { + var dc dummyContainer + dc.serviceName = "foo" + dc.jvmOpts = NewOptions(&dc) + return &dc +} + +func TestHeapSizeSimple(t *testing.T) { + var ( + aa = MegaBytesOfMemory(123) + bb = MegaBytesOfMemory(234) + ) + o := newDummyContainer().JvmOptions() + assert.Equal(t, aa, o.CurMinHeapSize(aa)) + assert.Equal(t, bb, o.CurMaxHeapSize(bb)) + assert.Equal(t, 2, len(o.jvmArgs)) + assert.Equal(t, "-Xms123m", o.jvmArgs[0]) + assert.Equal(t, "-Xmx234m", o.jvmArgs[1]) +} + +func TestHeapSizeMulti(t *testing.T) { + var ( + aa = MegaBytesOfMemory(123) + bb = MegaBytesOfMemory(234) + cc = MegaBytesOfMemory(456) + dd = MegaBytesOfMemory(567) + ) + o := newDummyContainer().JvmOptions() + assert.Equal(t, aa, o.CurMinHeapSize(aa)) + assert.Equal(t, aa, o.CurMaxHeapSize(aa)) + assert.Equal(t, 2, len(o.jvmArgs)) + o.AppendOption("-Xms234m") + o.AppendOption("-Xmx456m") + assert.Equal(t, 4, len(o.jvmArgs)) + assert.Equal(t, bb, o.CurMinHeapSize(aa)) + assert.Equal(t, bb, o.CurMinHeapSize(dd)) + assert.Equal(t, cc, o.CurMaxHeapSize(aa)) + assert.Equal(t, cc, o.CurMaxHeapSize(dd)) + o.AppendOption("-Xms1g") + o.AppendOption("-Xmx2g") + assert.Equal(t, GigaBytesOfMemory(1), o.CurMinHeapSize(aa)) + assert.Equal(t, GigaBytesOfMemory(2), o.CurMaxHeapSize(aa)) + o.AppendOption("-Xms16777216k") + o.AppendOption("-Xmx32505856k") + assert.Equal(t, KiloBytesOfMemory(16777216), o.CurMinHeapSize(aa)) + assert.Equal(t, KiloBytesOfMemory(32505856), o.CurMaxHeapSize(aa)) +} + +func TestHeapSizeAdd(t *testing.T) { + var ( + gg = MegaBytesOfMemory(12345) + hh = MegaBytesOfMemory(23456) + ) + o := newDummyContainer().JvmOptions() + o.AddDefaultHeapSizeArgs(gg, hh) + assert.Equal(t, 3, len(o.jvmArgs)) + assert.Equal(t, "-Xms12345m", o.jvmArgs[0]) + assert.Equal(t, "-Xmx23456m", o.jvmArgs[1]) + assert.Equal(t, "-XX:+UseTransparentHugePages", o.jvmArgs[2]) +} + +func TestHeapSizeNoAdd(t *testing.T) { + var ( + bb = MegaBytesOfMemory(234) + cc = MegaBytesOfMemory(456) + ) + o := newDummyContainer().JvmOptions() + o.AppendOption("-Xms128k") + o.AppendOption("-Xmx1280k") + o.AddDefaultHeapSizeArgs(bb, cc) + assert.Equal(t, 2, len(o.jvmArgs)) + assert.Equal(t, "-Xms128k", o.jvmArgs[0]) + assert.Equal(t, "-Xmx1280k", o.jvmArgs[1]) +} diff --git a/client/go/internal/admin/jvm/properties.go b/client/go/internal/admin/jvm/properties.go new file mode 100644 index 00000000000..9aae550b10a --- /dev/null +++ b/client/go/internal/admin/jvm/properties.go @@ -0,0 +1,106 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "bytes" + "fmt" + "os" + "sort" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +// quote as specified in JDK source file java.base/share/classes/java/util/Properties.java +func propQuote(s string, buf *bytes.Buffer) { + inKey := true + inVal := false + for _, ch := range s { + needQ := false + switch { + case ch == ' ': + needQ = !inVal + case ch == ':': + needQ = inKey + case ch < ' ': + needQ = true + case ch > '~': + needQ = true + case ch == '\\': + buf.WriteString("\\") + case ch == '=': + inKey = false + default: + inVal = !inKey + } + if needQ { + fmt.Fprintf(buf, "\\u%04X", ch) + } else { + buf.WriteRune(ch) + } + } +} + +func envAsProperties(envv []string) []byte { + suppress := map[string]bool{ + "_": true, + "HISTCONTROL": true, + "HISTSIZE": true, + "IFS": true, + "LESSOPEN": true, + "LOADEDMODULES": true, + "LS_COLORS": true, + "MAIL": true, + "MODULEPATH": true, + "MODULEPATH_modshare": true, + "MODULESHOME": true, + "MODULES_CMD": true, + "MODULES_RUN_QUARANTINE": true, + "OLDPWD": true, + "PCP_DIR": true, + "PS1": true, + "PWD": true, + "SHLVL": true, + "SSH_AUTH_SOCK": true, + "SSH_CLIENT": true, + "SSH_CONNECTION": true, + "SSH_TTY": true, + "S_COLORS": true, + "which_declare": true, + "": true, + } + var buf bytes.Buffer + buf.WriteString("# properties converted from environment variables\n") + sort.Strings(envv) + for _, env := range envv { + parts := strings.Split(env, "=") + if len(parts) >= 2 { + varName := parts[0] + if suppress[varName] || strings.Contains(varName, "%%") { + continue + } + if strings.Contains(env, "\n") { + continue + } + propQuote(env, &buf) + buf.WriteRune('\n') + } else { + trace.Warning("environment value without '=':", env) + } + } + return buf.Bytes() +} + +func writeEnvAsProperties(envv []string, propsFile string) { + if propsFile == "" { + panic("missing propsFile") + } + trace.Trace("write props file:", propsFile) + err := os.WriteFile(propsFile, envAsProperties(envv), 0600) + if err != nil { + util.JustExitWith(err) + } +} diff --git a/client/go/internal/admin/jvm/properties_test.go b/client/go/internal/admin/jvm/properties_test.go new file mode 100644 index 00000000000..c417319a51e --- /dev/null +++ b/client/go/internal/admin/jvm/properties_test.go @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package jvm + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSomeQuoting(t *testing.T) { + var buf bytes.Buffer + propQuote("a=b", &buf) + assert.Equal(t, "a=b", buf.String()) + buf.Reset() + propQuote("foobar", &buf) + assert.Equal(t, "foobar", buf.String()) + buf.Reset() + propQuote("x y = foo bar ", &buf) + assert.Equal(t, `x\u0020y\u0020=\u0020foo bar `, buf.String()) + buf.Reset() + propQuote("x:y=foo:bar", &buf) + assert.Equal(t, "x\\u003Ay=foo:bar", buf.String()) + buf.Reset() + propQuote(`PS1=\[\e[0;32m\]AWS-US\[\e[0m\] [\u@\[\e[1m\]\h\[\e[0m\]:\w]$ `, &buf) + assert.Equal(t, `PS1=\\[\\e[0;32m\\]AWS-US\\[\\e[0m\\] [\\u@\\[\\e[1m\\]\\h\\[\\e[0m\\]:\\w]$ `, buf.String()) +} diff --git a/client/go/internal/admin/jvm/qr_start_cfg.go b/client/go/internal/admin/jvm/qr_start_cfg.go new file mode 100644 index 00000000000..86e05c64418 --- /dev/null +++ b/client/go/internal/admin/jvm/qr_start_cfg.go @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "encoding/json" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +type QrStartConfig struct { + Jvm struct { + Server bool `json:"server"` + Verbosegc bool `json:"verbosegc"` + Gcopts string `json:"gcopts"` + Heapsize int `json:"heapsize"` + MinHeapsize int `json:"minHeapsize"` + Stacksize int `json:"stacksize"` + CompressedClassSpaceSize int `json:"compressedClassSpaceSize"` + BaseMaxDirectMemorySize int `json:"baseMaxDirectMemorySize"` + DirectMemorySizeCache int `json:"directMemorySizeCache"` + HeapSizeAsPercentageOfPhysicalMemory int `json:"heapSizeAsPercentageOfPhysicalMemory"` + AvailableProcessors int `json:"availableProcessors"` + } `json:"jvm"` + Qrs struct { + Env string `json:"env"` + } `json:"qrs"` + Jdisc struct { + ClasspathExtra string `json:"classpath_extra"` + ExportPackages string `json:"export_packages"` + } `json:"jdisc"` +} + +func (a *ApplicationContainer) getQrStartCfg() *QrStartConfig { + var parsedJson QrStartConfig + args := []string{ + "-j", + "-w", "10", + "-n", "search.config.qr-start", + "-i", a.ConfigId(), + } + backticks := util.BackTicksForwardStderr + data, err := backticks.Run("vespa-get-config", args...) + if err != nil { + trace.Trace("could not get qr-start config:", err) + } else { + codec := json.NewDecoder(strings.NewReader(data)) + err = codec.Decode(&parsedJson) + if err != nil { + trace.Trace("could not decode JSON >>>", data, "<<< error:", err) + } + } + return &parsedJson +} diff --git a/client/go/internal/admin/jvm/run.go b/client/go/internal/admin/jvm/run.go new file mode 100644 index 00000000000..30d5b61a063 --- /dev/null +++ b/client/go/internal/admin/jvm/run.go @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +func RunApplicationContainer(extraArgs []string) int { + if doTrace := os.Getenv(envvars.TRACE_JVM_STARTUP); doTrace != "" { + trace.AdjustVerbosity(1) + } + if doDebug := os.Getenv(envvars.DEBUG_JVM_STARTUP); doDebug != "" { + trace.AdjustVerbosity(2) + } + container := NewApplicationContainer(extraArgs) + container.Exec() + // unreachable: + return 1 +} + +func NewApplicationContainer(extraArgs []string) Container { + var a ApplicationContainer + a.configId = os.Getenv(envvars.VESPA_CONFIG_ID) + a.serviceName = os.Getenv(envvars.VESPA_SERVICE_NAME) + a.jvmOpts = NewOptions(&a) + a.configureOptions() + for _, x := range extraArgs { + a.JvmOptions().AddOption(x) + } + return &a +} diff --git a/client/go/internal/admin/jvm/standalone_container.go b/client/go/internal/admin/jvm/standalone_container.go new file mode 100644 index 00000000000..7bbf03e83e8 --- /dev/null +++ b/client/go/internal/admin/jvm/standalone_container.go @@ -0,0 +1,86 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/prog" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const ( + JAR_FOR_STANDALONE_CONTAINER = "standalone-container-jar-with-dependencies.jar" +) + +type StandaloneContainer struct { + containerBase +} + +func (a *StandaloneContainer) ArgForMain() string { + return JAR_FOR_STANDALONE_CONTAINER +} + +func (a *StandaloneContainer) ServiceName() string { + return a.serviceName +} + +func (a *StandaloneContainer) ConfigId() string { + return "" +} + +func (a *StandaloneContainer) configureOptions() { + opts := a.jvmOpts + opts.ConfigureCpuCount(0) + opts.AddCommonXX() + opts.AddOption("-XX:-OmitStackTraceInFastThrow") + opts.AddCommonOpens() + opts.AddCommonJdkProperties() + a.addJdiscProperties() + svcName := a.ServiceName() + if svcName == "configserver" { + RemoveStaleZkLocks(a) + logsDir := defaults.UnderVespaHome("logs/vespa") + zkLogFile := fmt.Sprintf("%s/zookeeper.%s", logsDir, svcName) + opts.AddOption("-Dzookeeper_log_file_prefix=" + zkLogFile) + } +} + +func NewStandaloneContainer(svcName string) Container { + var a StandaloneContainer + a.serviceName = svcName + a.jvmOpts = NewOptions(&a) + a.configureOptions() + return &a +} + +func (a *StandaloneContainer) addJdiscProperties() { + opts := a.JvmOptions() + opts.AddCommonJdiscProperties() + containerParentDir := defaults.UnderVespaHome("var/jdisc_container") + bCacheParentDir := defaults.UnderVespaHome("var/vespa/bundlecache") + svcName := a.ServiceName() + bCacheDir := fmt.Sprintf("%s/%s", bCacheParentDir, svcName) + propsFile := fmt.Sprintf("%s/%s.properties", containerParentDir, svcName) + opts.fixSpec.FixDir(containerParentDir) + opts.fixSpec.FixDir(bCacheParentDir) + opts.fixSpec.FixDir(bCacheDir) + a.propsFile = propsFile + opts.AddOption("-Djdisc.export.packages=") + opts.AddOption("-Djdisc.config.file=" + propsFile) + opts.AddOption("-Djdisc.cache.path=" + bCacheDir) + opts.AddOption("-Djdisc.logger.tag=" + svcName) +} + +func (c *StandaloneContainer) exportExtraEnv(ps *prog.Spec) { + vespaHome := defaults.VespaHome() + app := fmt.Sprintf("%s/conf/%s-app", vespaHome, c.ServiceName()) + if util.IsDirectory(app) { + ps.Setenv(envvars.STANDALONE_JDISC_APP_LOCATION, app) + } else { + util.JustExitMsg("standalone container requires an application directory, missing: " + app) + } +} diff --git a/client/go/internal/admin/jvm/xx_options.go b/client/go/internal/admin/jvm/xx_options.go new file mode 100644 index 00000000000..ea984df1dfe --- /dev/null +++ b/client/go/internal/admin/jvm/xx_options.go @@ -0,0 +1,21 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" +) + +func (opts *Options) AddCommonXX() { + crashDir := defaults.UnderVespaHome("var/crash") + errorFile := crashDir + "/hs_err_pid%p.log" + opts.fixSpec.FixDir(crashDir) + opts.AddOption("-XX:+PreserveFramePointer") + opts.AddOption("-XX:+HeapDumpOnOutOfMemoryError") + opts.AddOption("-XX:HeapDumpPath=" + crashDir) + opts.AddOption("-XX:ErrorFile=" + errorFile) + opts.AddOption("-XX:+ExitOnOutOfMemoryError") + // not common after all: + opts.AddOption("-XX:MaxJavaStackTraceDepth=1000000") +} diff --git a/client/go/internal/admin/jvm/zk_locks.go b/client/go/internal/admin/jvm/zk_locks.go new file mode 100644 index 00000000000..39b8dd0c64b --- /dev/null +++ b/client/go/internal/admin/jvm/zk_locks.go @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package jvm + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const ( + ZOOKEEPER_LOG_FILE_PREFIX = "logs/vespa/zookeeper" +) + +func RemoveStaleZkLocks(c Container) { + backticks := util.BackTicksWithStderr + cmd := fmt.Sprintf("rm -f '%s/%s.%s'*lck", defaults.VespaHome(), ZOOKEEPER_LOG_FILE_PREFIX, c.ServiceName()) + trace.Trace("cleaning locks:", cmd) + out, err := backticks.Run("/bin/sh", "-c", cmd) + if err != nil { + trace.Warning("Failure [", out, "] when running command:", cmd) + util.JustExitWith(err) + } +} diff --git a/client/go/internal/admin/prog/common_env.go b/client/go/internal/admin/prog/common_env.go new file mode 100644 index 00000000000..f743716a64e --- /dev/null +++ b/client/go/internal/admin/prog/common_env.go @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package prog + +import ( + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func (spec *Spec) configureCommonEnv() { + os.Unsetenv(envvars.LD_PRELOAD) + spec.Setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") + spec.Setenv(envvars.GLIBCXX_FORCE_NEW, "1") + spec.Setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") + spec.Setenv(envvars.MALLOC_ARENA_MAX, "1") + + // fallback from old env.vars: + spec.considerEnvFallback(envvars.VESPA_USE_HUGEPAGES_LIST, envvars.HUGEPAGES_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_MADVISE_LIST, envvars.MADVISE_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC, envvars.VESPAMALLOC_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_D, envvars.VESPAMALLOCD_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_DST, envvars.VESPAMALLOCDST_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_NO_VESPAMALLOC, envvars.NO_VESPAMALLOC_LIST) + // other fallbacks: + spec.considerFallback(envvars.ROOT, vespa.FindHome()) + spec.considerFallback(envvars.VESPA_USER, vespa.FindVespaUser()) + spec.considerFallback(envvars.VESPA_USE_HUGEPAGES_LIST, "all") + spec.considerFallback(envvars.VESPA_USE_VESPAMALLOC, "all") + spec.considerFallback(envvars.VESPA_USE_NO_VESPAMALLOC, strings.Join([]string{ + "vespa-rpc-invoke", + "vespa-get-config", + "vespa-sentinel-cmd", + "vespa-route", + "vespa-proton-cmd", + "vespa-configproxy-cmd", + "vespa-config-status", + }, " ")) + +} diff --git a/client/go/internal/admin/prog/hugepages.go b/client/go/internal/admin/prog/hugepages.go new file mode 100644 index 00000000000..c6f019937ff --- /dev/null +++ b/client/go/internal/admin/prog/hugepages.go @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package prog + +import ( + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +func (spec *Spec) ConfigureHugePages() { + if spec.matchesListEnv(envvars.VESPA_USE_HUGEPAGES_LIST) { + trace.Debug("setting", envvars.VESPA_USE_HUGEPAGES, "= 'yes'") + spec.Setenv(envvars.VESPA_USE_HUGEPAGES, "yes") + } +} diff --git a/client/go/internal/admin/prog/madvise.go b/client/go/internal/admin/prog/madvise.go new file mode 100644 index 00000000000..48986a12182 --- /dev/null +++ b/client/go/internal/admin/prog/madvise.go @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package prog + +import ( + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +func (spec *Spec) ConfigureUseMadvise() { + limit := spec.valueFromListEnv(envvars.VESPA_USE_MADVISE_LIST) + if limit != "" { + trace.Trace("shall use madvise with limit", limit, "as set in", envvars.VESPA_USE_MADVISE_LIST) + spec.Setenv(envvars.VESPA_MALLOC_MADVISE_LIMIT, limit) + return + } +} diff --git a/client/go/internal/admin/prog/mockbin/bad-numactl b/client/go/internal/admin/prog/mockbin/bad-numactl new file mode 100755 index 00000000000..1b08bb19b95 --- /dev/null +++ b/client/go/internal/admin/prog/mockbin/bad-numactl @@ -0,0 +1,25 @@ +#!/bin/sh + +# emulate numactl which says hardware is available but fails anyway + +case $#.$* in + 1.--hardware) + cat << 'EOF' +available: 2 nodes (0-1) +node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 +node 0 size: 128604 MB +node 0 free: 107167 MB +node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39 41 43 45 47 +node 1 size: 128976 MB +node 1 free: 110921 MB +node distances: +node 0 1 + 0: 10 21 + 1: 21 10 +EOF + ;; + *) + echo "numactl failure" + ;; +esac +exit 0 diff --git a/client/go/internal/admin/prog/mockbin/good-numactl b/client/go/internal/admin/prog/mockbin/good-numactl new file mode 100755 index 00000000000..f861809b382 --- /dev/null +++ b/client/go/internal/admin/prog/mockbin/good-numactl @@ -0,0 +1,25 @@ +#!/bin/sh + +# emulate normal numactl operations + +case $#.$* in + 1.--hardware) + cat << 'EOF' +available: 2 nodes (0-1) +node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 +node 0 size: 128604 MB +node 0 free: 107167 MB +node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39 41 43 45 47 +node 1 size: 128976 MB +node 1 free: 110921 MB +node distances: +node 0 1 + 0: 10 21 + 1: 21 10 +EOF + ;; + *) + echo "foo" + ;; +esac +exit 0 diff --git a/client/go/internal/admin/prog/mockbin/has-valgrind b/client/go/internal/admin/prog/mockbin/has-valgrind new file mode 100755 index 00000000000..83601cd64d1 --- /dev/null +++ b/client/go/internal/admin/prog/mockbin/has-valgrind @@ -0,0 +1,6 @@ +#!/bin/sh + +# emulate working valgrind + +echo "some help text here" +exit 0 diff --git a/client/go/internal/admin/prog/mockbin/no-numactl b/client/go/internal/admin/prog/mockbin/no-numactl new file mode 100755 index 00000000000..dfe583abaca --- /dev/null +++ b/client/go/internal/admin/prog/mockbin/no-numactl @@ -0,0 +1,6 @@ +#!/bin/sh + +# emulate inoperational numactl + +echo "No NUMA available on this system" +exit 1 diff --git a/client/go/internal/admin/prog/mockbin/no-valgrind b/client/go/internal/admin/prog/mockbin/no-valgrind new file mode 100755 index 00000000000..09a2e517c37 --- /dev/null +++ b/client/go/internal/admin/prog/mockbin/no-valgrind @@ -0,0 +1,6 @@ +#!/bin/sh + +# emulate bad or missing valgrind + +echo "some error message here" +exit 1 diff --git a/client/go/internal/admin/prog/numactl.go b/client/go/internal/admin/prog/numactl.go new file mode 100644 index 00000000000..e53b4feb3d8 --- /dev/null +++ b/client/go/internal/admin/prog/numactl.go @@ -0,0 +1,74 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package prog + +import ( + "fmt" + "strconv" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const ( + NUMACTL_PROG = "numactl" +) + +func (p *Spec) ConfigureNumaCtl() { + p.shouldUseNumaCtl = false + p.numaSocket = -1 + if p.Getenv(envvars.VESPA_NO_NUMACTL) != "" { + return + } + backticks := util.BackTicksIgnoreStderr + out, err := backticks.Run(NUMACTL_PROG, "--hardware") + trace.Debug("numactl --hardware says:", out) + if err != nil { + trace.Trace("numactl error:", err) + return + } + outfoo, errfoo := backticks.Run(NUMACTL_PROG, "--interleave", "all", "echo", "foo") + if errfoo != nil { + trace.Trace("cannot run with numactl:", errfoo) + return + } + if outfoo != "foo\n" { + trace.Trace("bad numactl output:", outfoo) + return + } + p.shouldUseNumaCtl = true + if affinity := p.Getenv(envvars.VESPA_AFFINITY_CPU_SOCKET); affinity != "" { + wantSocket, _ := strconv.Atoi(affinity) + trace.Debug("want socket:", wantSocket) + parts := strings.Fields(out) + for idx := 0; idx+2 < len(parts); idx++ { + if parts[idx] == "available:" && parts[idx+2] == "nodes" { + numSockets, _ := strconv.Atoi(parts[idx+1]) + trace.Debug("numSockets:", numSockets) + if numSockets > 1 { + p.numaSocket = (wantSocket % numSockets) + return + } + } + } + } +} + +func (p *Spec) prependNumaCtl(args []string) []string { + result := make([]string, 0, 5+len(args)) + result = append(result, NUMACTL_PROG) + if p.numaSocket >= 0 { + result = append(result, fmt.Sprintf("--cpunodebind=%d", p.numaSocket)) + result = append(result, fmt.Sprintf("--membind=%d", p.numaSocket)) + } else { + result = append(result, "--interleave") + result = append(result, "all") + } + for _, arg := range args { + result = append(result, arg) + } + return result +} diff --git a/client/go/internal/admin/prog/numactl_test.go b/client/go/internal/admin/prog/numactl_test.go new file mode 100644 index 00000000000..569219f5b7c --- /dev/null +++ b/client/go/internal/admin/prog/numactl_test.go @@ -0,0 +1,90 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package prog + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +func setup(t *testing.T, testFileName string) { + trace.AdjustVerbosity(1) + mockBinParent = strings.TrimSuffix(testFileName, "/numactl_test.go") + tmpBin = t.TempDir() + "/mock.bin.numactl_test" + err := os.MkdirAll(tmpBin, 0755) + assert.Nil(t, err) + t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH"))) +} + +func TestNumaCtlDetection(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + _, tfn, _, _ := runtime.Caller(0) + setup(t, tfn) + orig := []string{"/bin/myprog", "-c", "cfgid"} + spec := NewSpec(orig) + + useMock("no-numactl", "numactl") + spec.ConfigureNumaCtl() + assert.Equal(t, false, spec.shouldUseNumaCtl) + + useMock("bad-numactl", "numactl") + spec.ConfigureNumaCtl() + assert.Equal(t, false, spec.shouldUseNumaCtl) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "") + useMock("good-numactl", "numactl") + spec.ConfigureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, -1, spec.numaSocket) + argv := spec.prependNumaCtl(orig) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "numactl", argv[0]) + assert.Equal(t, "--interleave", argv[1]) + assert.Equal(t, "all", argv[2]) + assert.Equal(t, "/bin/myprog", argv[3]) + assert.Equal(t, "-c", argv[4]) + assert.Equal(t, "cfgid", argv[5]) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "0") + spec.ConfigureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, 0, spec.numaSocket) + argv = spec.prependNumaCtl(orig) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "numactl", argv[0]) + assert.Equal(t, "--cpunodebind=0", argv[1]) + assert.Equal(t, "--membind=0", argv[2]) + assert.Equal(t, "/bin/myprog", argv[3]) + assert.Equal(t, "-c", argv[4]) + assert.Equal(t, "cfgid", argv[5]) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "1") + spec.ConfigureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, 1, spec.numaSocket) + argv = spec.prependNumaCtl(orig) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "numactl", argv[0]) + assert.Equal(t, "--cpunodebind=1", argv[1]) + assert.Equal(t, "--membind=1", argv[2]) + assert.Equal(t, "/bin/myprog", argv[3]) + assert.Equal(t, "-c", argv[4]) + assert.Equal(t, "cfgid", argv[5]) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "2") + spec.ConfigureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, 0, spec.numaSocket) + +} diff --git a/client/go/internal/admin/prog/run.go b/client/go/internal/admin/prog/run.go new file mode 100644 index 00000000000..f1a2e979600 --- /dev/null +++ b/client/go/internal/admin/prog/run.go @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package prog + +import ( + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func (spec *Spec) Run() error { + prog := spec.Program + args := spec.Args + if spec.shouldUseValgrind { + args = spec.prependValgrind(args) + prog = args[0] + } else if spec.shouldUseNumaCtl { + args = spec.prependNumaCtl(args) + prog = args[0] + } + if spec.shouldUseVespaMalloc { + spec.Setenv(envvars.LD_PRELOAD, spec.vespaMallocPreload) + } + envv := spec.EffectiveEnv() + return util.Execvpe(prog, args, envv) +} diff --git a/client/go/internal/admin/prog/spec.go b/client/go/internal/admin/prog/spec.go new file mode 100644 index 00000000000..cf553470a44 --- /dev/null +++ b/client/go/internal/admin/prog/spec.go @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package prog + +import ( + "strings" +) + +type Spec struct { + Program string + Args []string + BaseName string + Env map[string]string + numaSocket int + shouldUseCallgrind bool + shouldUseValgrind bool + shouldUseNumaCtl bool + shouldUseVespaMalloc bool + vespaMallocPreload string +} + +func NewSpec(argv []string) *Spec { + progName := argv[0] + p := Spec{ + Program: progName, + Args: argv, + BaseName: baseNameOf(progName), + Env: make(map[string]string), + numaSocket: -1, + } + return &p +} + +func baseNameOf(s string) string { + idx := strings.LastIndex(s, "/") + idx++ + return s[idx:] +} diff --git a/client/go/internal/admin/prog/spec_env.go b/client/go/internal/admin/prog/spec_env.go new file mode 100644 index 00000000000..4fa40695acb --- /dev/null +++ b/client/go/internal/admin/prog/spec_env.go @@ -0,0 +1,105 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package prog + +import ( + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +func (p *Spec) Setenv(k, v string) { + p.Env[k] = v +} + +func (p *Spec) Getenv(k string) string { + if v, ok := p.Env[k]; ok { + return v + } + return os.Getenv(k) +} + +func (spec *Spec) EffectiveEnv() []string { + envMap := make(map[string]string) + addToMap := func(kv string) { + for idx, elem := range kv { + if elem == '=' { + k := kv[:idx] + envMap[k] = kv + return + } + } + trace.Trace("invalid entry in os.Environ():", kv) + envMap[kv] = kv + } + for _, entry := range os.Environ() { + addToMap(entry) + } + for k, v := range spec.Env { + trace.Trace("add to environment:", k, "=", v) + envMap[k] = k + "=" + v + } + envVec := make([]string, len(envMap)) + idx := 0 + for _, val := range envMap { + envVec[idx] = val + idx++ + } + return envVec +} + +func (spec *Spec) considerFallback(varName, varValue string) { + if spec.Getenv(varName) == "" && varValue != "" { + spec.Setenv(varName, varValue) + } +} + +func (spec *Spec) considerEnvFallback(targetVar, fallbackVar string) { + spec.considerFallback(targetVar, spec.Getenv(fallbackVar)) +} + +func (p *Spec) matchesListEnv(envVarName string) bool { + return p.matchesListString(p.Getenv(envVarName)) +} + +func (p *Spec) matchesListString(env string) bool { + if env == "all" { + trace.Debug(p.Program, "always matching in:", env) + return true + } + parts := strings.Fields(env) + for _, part := range parts { + if p.BaseName == part { + trace.Debug(p.Program, "has basename matching in:", env) + return true + } + trace.Debug("checking matching:", p.BaseName, "!=", part) + } + return false +} + +func (p *Spec) valueFromListEnv(envVarName string) string { + return p.valueFromListString(p.Getenv(envVarName)) +} + +func (p *Spec) valueFromListString(env string) string { + parts := strings.Fields(env) + for _, part := range parts { + idx := strings.Index(part, "=") + if idx <= 0 { + trace.Trace("expected key=value, but got:", part) + continue + } + partName := part[:idx] + idx++ + partValue := part[idx:] + if p.BaseName == partName || partName == "all" { + trace.Debug(p.Program, "has basename matching in:", env) + return partValue + } + trace.Debug("checking matching:", p.BaseName, "!=", part) + } + return "" +} diff --git a/client/go/internal/admin/prog/spec_test.go b/client/go/internal/admin/prog/spec_test.go new file mode 100644 index 00000000000..0e5d3fb50ba --- /dev/null +++ b/client/go/internal/admin/prog/spec_test.go @@ -0,0 +1,63 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package prog + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProgSpec(t *testing.T) { + spec := NewSpec([]string{"/opt/vespa/bin/foobar"}) + var b bool + + b = spec.matchesListString("") + assert.Equal(t, false, b) + b = spec.matchesListString("foobar") + assert.Equal(t, true, b) + b = spec.matchesListString("foo bar") + assert.Equal(t, false, b) + b = spec.matchesListString("one foobar") + assert.Equal(t, true, b) + b = spec.matchesListString("foobar two") + assert.Equal(t, true, b) + b = spec.matchesListString("one foobar two") + assert.Equal(t, true, b) + b = spec.matchesListString("all") + assert.Equal(t, true, b) + + var s string + s = spec.valueFromListString("") + assert.Equal(t, "", s) + s = spec.valueFromListString("foobar=123") + assert.Equal(t, "123", s) + s = spec.valueFromListString("one=1 foobar=123 two=2") + assert.Equal(t, "123", s) + s = spec.valueFromListString("one=1 all=123") + assert.Equal(t, "123", s) +} + +type strVec []string + +func (v strVec) contains(w string) bool { + for _, val := range v { + if w == val { + return true + } + } + return false +} + +func TestProgSpecEnv(t *testing.T) { + spec := NewSpec([]string{"/opt/vespa/bin/foobar"}) + t.Setenv("FOO", "old foo") + t.Setenv("BAR", "bar") + spec.Setenv("FOO", "foo") + assert.Equal(t, "foo", spec.Getenv("FOO")) + assert.Equal(t, "bar", spec.Getenv("BAR")) + envv := strVec(spec.EffectiveEnv()) + assert.True(t, envv.contains("FOO=foo")) + assert.True(t, envv.contains("BAR=bar")) + assert.False(t, envv.contains("FOO=old foo")) +} diff --git a/client/go/internal/admin/prog/valgrind.go b/client/go/internal/admin/prog/valgrind.go new file mode 100644 index 00000000000..7d3fb059f8f --- /dev/null +++ b/client/go/internal/admin/prog/valgrind.go @@ -0,0 +1,86 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package prog + +import ( + "fmt" + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + VALGRIND_PROG = "valgrind" +) + +func (p *Spec) ConfigureValgrind() { + p.shouldUseValgrind = false + p.shouldUseCallgrind = false + env := p.Getenv(envvars.VESPA_USE_VALGRIND) + parts := strings.Split(env, " ") + for _, part := range parts { + if p.BaseName == part { + trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env) + backticks := util.BackTicksWithStderr + out, err := backticks.Run(VALGRIND_PROG, "--help") + if err != nil { + trace.Trace("trial run of valgrind fails:", err, "=>", out) + return + } + if opts := p.Getenv(envvars.VESPA_VALGRIND_OPT); strings.Contains(opts, "callgrind") { + p.shouldUseCallgrind = true + } + p.shouldUseValgrind = true + return + } + trace.Debug("checking", envvars.VESPA_USE_VALGRIND, ":", p.BaseName, "!=", part) + } +} + +func (p *Spec) valgrindOptions() []string { + env := p.Getenv(envvars.VESPA_VALGRIND_OPT) + if env != "" { + return strings.Fields(env) + } + result := []string{ + "--num-callers=32", + "--run-libc-freeres=yes", + "--track-origins=yes", + "--freelist-vol=1000000000", + "--leak-check=full", + "--show-reachable=yes", + } + result = addValgrindSuppression(result, "etc/vespa/valgrind-suppressions.txt") + result = addValgrindSuppression(result, "etc/vespa/suppressions.txt") + return result +} + +func addValgrindSuppression(r []string, fn string) []string { + existsOk, fileName := vespa.HasFileUnderVespaHome(fn) + if existsOk { + r = append(r, fmt.Sprintf("--suppressions=%s", fileName)) + } + return r +} + +func (p *Spec) valgrindLogOption() string { + return fmt.Sprintf("--log-file=%s/var/tmp/valgrind.%s.log.%d", vespa.FindHome(), p.BaseName, os.Getpid()) +} + +func (p *Spec) prependValgrind(args []string) []string { + result := make([]string, 0, 15+len(args)) + result = append(result, VALGRIND_PROG) + for _, arg := range p.valgrindOptions() { + result = append(result, arg) + } + result = append(result, p.valgrindLogOption()) + for _, arg := range args { + result = append(result, arg) + } + return result +} diff --git a/client/go/internal/admin/prog/valgrind_test.go b/client/go/internal/admin/prog/valgrind_test.go new file mode 100644 index 00000000000..6ec622277c6 --- /dev/null +++ b/client/go/internal/admin/prog/valgrind_test.go @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package prog + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +var tmpBin string +var mockBinParent string + +func useMock(prog, target string) { + mock := fmt.Sprintf("%s/mockbin/%s", mockBinParent, prog) + symlink := fmt.Sprintf("%s/%s", tmpBin, target) + os.Remove(symlink) + err := os.Symlink(mock, symlink) + if err != nil { + util.JustExitWith(err) + } +} + +func setupValgrind(t *testing.T, testFileName string) { + trace.AdjustVerbosity(1) + t.Setenv("VESPA_HOME", mockBinParent+"/mock_vespahome") + mockBinParent = strings.TrimSuffix(testFileName, "/valgrind_test.go") + tmpBin = t.TempDir() + "/mock.bin.valgrind_test" + err := os.MkdirAll(tmpBin, 0755) + assert.Nil(t, err) + t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH"))) +} + +func TestValgrindDetection(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + _, tfn, _, _ := runtime.Caller(0) + setupValgrind(t, tfn) + spec := NewSpec([]string{"/opt/vespa/bin/foobar"}) + var argv []string + + useMock("has-valgrind", VALGRIND_PROG) + + t.Setenv("VESPA_USE_VALGRIND", "") + spec.ConfigureValgrind() + assert.Equal(t, false, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + t.Setenv("VESPA_USE_VALGRIND", "foo bar") + spec.ConfigureValgrind() + assert.Equal(t, false, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + t.Setenv("VESPA_USE_VALGRIND", "foobar") + spec.ConfigureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"}) + trace.Trace("argv:", argv) + assert.Equal(t, 11, len(argv)) + assert.Equal(t, "valgrind", argv[0]) + assert.Equal(t, "/bin/myprog", argv[8]) + + t.Setenv("VESPA_USE_VALGRIND", "another foobar yetmore") + spec.ConfigureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + t.Setenv("VESPA_VALGRIND_OPT", "--tool=callgrind") + spec.ConfigureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, true, spec.shouldUseCallgrind) + + argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"}) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "valgrind", argv[0]) + assert.Equal(t, "--tool=callgrind", argv[1]) + assert.Equal(t, "/bin/myprog", argv[3]) + + useMock("no-valgrind", "valgrind") + spec.ConfigureValgrind() + assert.Equal(t, false, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) +} diff --git a/client/go/internal/admin/prog/vespamalloc.go b/client/go/internal/admin/prog/vespamalloc.go new file mode 100644 index 00000000000..439935770d7 --- /dev/null +++ b/client/go/internal/admin/prog/vespamalloc.go @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package prog + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func vespaMallocLib(suf string) string { + prefixes := []string{"lib64", "lib"} + for _, pre := range prefixes { + fn := fmt.Sprintf("%s/vespa/malloc/%s", pre, suf) + existsOk, fileName := vespa.HasFileUnderVespaHome(fn) + if existsOk { + trace.Debug("found library:", fileName) + return fileName + } + trace.Debug("bad or missing library:", fn) + } + return "" +} + +func (p *Spec) ConfigureVespaMalloc() { + p.shouldUseVespaMalloc = false + if p.matchesListEnv(envvars.VESPA_USE_NO_VESPAMALLOC) { + trace.Trace("use no vespamalloc:", p.BaseName) + return + } + if p.shouldUseValgrind && !p.shouldUseCallgrind { + trace.Trace("use valgrind, so no vespamalloc:", p.BaseName) + return + } + var useFile string + if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_DST) { + useFile = vespaMallocLib("libvespamallocdst16.so") + } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_D) { + useFile = vespaMallocLib("libvespamallocd.so") + } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC) { + useFile = vespaMallocLib("libvespamalloc.so") + } + trace.Trace("use file:", useFile) + if useFile == "" { + return + } + if loadAsHuge := p.Getenv(envvars.VESPA_LOAD_CODE_AS_HUGEPAGES); loadAsHuge != "" { + otherFile := vespaMallocLib("libvespa_load_as_huge.so") + useFile = fmt.Sprintf("%s:%s", useFile, otherFile) + } + p.considerEnvFallback(envvars.VESPA_MALLOC_HUGEPAGES, envvars.VESPA_USE_HUGEPAGES) + p.vespaMallocPreload = useFile + p.shouldUseVespaMalloc = true +} diff --git a/client/go/internal/admin/script-utils/configserver/check.go b/client/go/internal/admin/script-utils/configserver/check.go new file mode 100644 index 00000000000..a0248dd128f --- /dev/null +++ b/client/go/internal/admin/script-utils/configserver/check.go @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func checkIsConfigserver(myname string) { + onlyHosts := defaults.VespaConfigserverHosts() + for _, hn := range onlyHosts { + if hn == "localhost" || hn == myname { + trace.Debug("should run configserver:", hn) + return + } + } + trace.Warning("only these hosts should run a config server:", onlyHosts) + util.JustExitMsg(fmt.Sprintf("this host [%s] should not run a config server", myname)) +} diff --git a/client/go/internal/admin/script-utils/configserver/env.go b/client/go/internal/admin/script-utils/configserver/env.go new file mode 100644 index 00000000000..1c4c33f5628 --- /dev/null +++ b/client/go/internal/admin/script-utils/configserver/env.go @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func exportSettings(vespaHome string) { + vlt := fmt.Sprintf("file:%s/logs/vespa/vespa.log", vespaHome) + lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome) + lcf := fmt.Sprintf("%s/configserver.logcontrol", lcd) + dlp := fmt.Sprintf("%s/lib64", vespaHome) + app := fmt.Sprintf("%s/conf/configserver-app", vespaHome) + os.Setenv(envvars.VESPA_LOG_TARGET, vlt) + os.Setenv(envvars.VESPA_LOG_CONTROL_DIR, lcd) + os.Setenv(envvars.VESPA_LOG_CONTROL_FILE, lcf) + os.Setenv(envvars.VESPA_SERVICE_NAME, "configserver") + os.Setenv(envvars.LD_LIBRARY_PATH, dlp) + os.Setenv(envvars.JAVAVM_LD_PRELOAD, "") + os.Setenv(envvars.LD_PRELOAD, "") + os.Setenv(envvars.STANDALONE_JDISC_APP_LOCATION, app) + os.Setenv(envvars.STANDALONE_JDISC_DEPLOYMENT_PROFILE, "configserver") + os.Setenv(envvars.MALLOC_ARENA_MAX, "1") + util.OptionallyReduceTimerFrequency() +} diff --git a/client/go/internal/admin/script-utils/configserver/fix_dirs_and_files.go b/client/go/internal/admin/script-utils/configserver/fix_dirs_and_files.go new file mode 100644 index 00000000000..8410d3f657c --- /dev/null +++ b/client/go/internal/admin/script-utils/configserver/fix_dirs_and_files.go @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func makeFixSpec() util.FixSpec { + vespaUid, vespaGid := vespa.FindVespaUidAndGid() + return util.FixSpec{ + UserId: vespaUid, + GroupId: vespaGid, + DirMode: 0755, + FileMode: 0644, + } +} + +func fixDirsAndFiles(fixSpec util.FixSpec) { + fixSpec.FixDir("var/zookeeper") + fixSpec.FixDir("var/zookeeper/conf") + fixSpec.FixDir("var/zookeeper/version-2") + fixSpec.FixFile("var/zookeeper/conf/zookeeper.cfg") + fixSpec.FixFile("var/zookeeper/myid") +} diff --git a/client/go/internal/admin/script-utils/configserver/logd.go b/client/go/internal/admin/script-utils/configserver/logd.go new file mode 100644 index 00000000000..39a2100b20f --- /dev/null +++ b/client/go/internal/admin/script-utils/configserver/logd.go @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func maybeStartLogd() { + v1 := os.Getenv(envvars.CLOUDCONFIG_SERVER_MULTITENANT) + v2 := os.Getenv(envvars.VESPA_CONFIGSERVER_MULTITENANT) + if v1 == "true" || v2 == "true" { + backticks := util.BackTicksForwardStderr + out, err := backticks.Run("libexec/vespa/start-logd") + if err != nil { + panic(err) + } + trace.Info(out) + } +} diff --git a/client/go/internal/admin/script-utils/configserver/runserver.go b/client/go/internal/admin/script-utils/configserver/runserver.go new file mode 100644 index 00000000000..bef50009d8d --- /dev/null +++ b/client/go/internal/admin/script-utils/configserver/runserver.go @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const ( + PROG_NAME = "vespa-runserver" +) + +type RunServer struct { + ServiceName string + Args []string +} + +func (rs *RunServer) PidFile() string { + varRunDir := defaults.UnderVespaHome("var/run") + return fmt.Sprintf("%s/%s.pid", varRunDir, rs.ServiceName) +} + +func (rs *RunServer) ProgPath() string { + p := fmt.Sprintf("%s/bin64/%s", defaults.VespaHome(), PROG_NAME) + if util.IsExecutableFile(p) { + return p + } + p = fmt.Sprintf("%s/bin/%s", defaults.VespaHome(), PROG_NAME) + if util.IsExecutableFile(p) { + return p + } + panic(fmt.Errorf("not an executable file: %s", p)) +} + +func (rs *RunServer) WouldRun() bool { + backticks := util.BackTicksForwardStderr + out, err := backticks.Run(rs.ProgPath(), "-s", rs.ServiceName, "-p", rs.PidFile(), "-W") + trace.Trace("output from -W:", out, "error:", err) + return err == nil +} + +func (rs *RunServer) Exec(prog string) { + argv := util.ArrayList[string]{ + PROG_NAME, + "-s", rs.ServiceName, + "-r", "30", + "-p", rs.PidFile(), + "--", + prog, + } + argv.AppendAll(rs.Args...) + err := util.Execvp(rs.ProgPath(), argv) + util.JustExitWith(err) +} diff --git a/client/go/internal/admin/script-utils/configserver/start.go b/client/go/internal/admin/script-utils/configserver/start.go new file mode 100644 index 00000000000..8ea3e9603a2 --- /dev/null +++ b/client/go/internal/admin/script-utils/configserver/start.go @@ -0,0 +1,86 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + SERVICE_NAME = "configserver" +) + +func commonPreChecks() (veHome string) { + if doTrace := os.Getenv(envvars.TRACE_JVM_STARTUP); doTrace != "" { + trace.AdjustVerbosity(1) + } + if doDebug := os.Getenv(envvars.DEBUG_JVM_STARTUP); doDebug != "" { + trace.AdjustVerbosity(2) + } + _ = vespa.FindAndVerifyVespaHome() + err := vespa.LoadDefaultEnv() + if err != nil { + panic(err) + } + veHome = vespa.FindAndVerifyVespaHome() + veHost, e := vespa.FindOurHostname() + if e != nil { + trace.Warning("could not detect hostname:", err, "; using fallback:", veHost) + } + checkIsConfigserver(veHost) + e = os.Chdir(veHome) + if e != nil { + util.JustExitWith(e) + } + return +} + +func JustStartConfigserver() int { + vespaHome := commonPreChecks() + vespa.CheckCorrectUser() + util.TuneResourceLimits() + util.TuneLogging(SERVICE_NAME, "com.google.api.client.http.HttpTransport", "config=off") + exportSettings(vespaHome) + removeStaleZkLocks(vespaHome) + c := jvm.NewStandaloneContainer(SERVICE_NAME) + jvmOpts := c.JvmOptions() + if extra := os.Getenv(envvars.VESPA_CONFIGSERVER_JVMARGS); extra != "" { + jvmOpts.AddJvmArgsFromString(extra) + } + minFallback := jvm.MegaBytesOfMemory(128) + maxFallback := jvm.MegaBytesOfMemory(2048) + jvmOpts.AddDefaultHeapSizeArgs(minFallback, maxFallback) + c.Exec() + // unreachable: + return 1 +} + +func runConfigserverWithRunserver() int { + commonPreChecks() + vespa.CheckCorrectUser() + rs := RunServer{ + ServiceName: SERVICE_NAME, + Args: []string{"just-start-configserver"}, + } + rs.Exec("libexec/vespa/script-utils") + return 1 +} + +func StartConfigserverEtc() int { + vespaHome := commonPreChecks() + vespa.RunPreStart() + util.TuneResourceLimits() + fixSpec := makeFixSpec() + fixDirsAndFiles(fixSpec) + exportSettings(vespaHome) + vespa.MaybeSwitchUser("vespa-start-configserver") + maybeStartLogd() + return runConfigserverWithRunserver() +} diff --git a/client/go/internal/admin/script-utils/configserver/zk.go b/client/go/internal/admin/script-utils/configserver/zk.go new file mode 100644 index 00000000000..91ddf522848 --- /dev/null +++ b/client/go/internal/admin/script-utils/configserver/zk.go @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const ( + ZOOKEEPER_LOG_FILE_PREFIX = "logs/vespa/zookeeper.configserver" +) + +func removeStaleZkLocks(vespaHome string) { + backticks := util.BackTicksIgnoreStderr + cmd := fmt.Sprintf("rm -f '%s/%s'*lck", vespaHome, ZOOKEEPER_LOG_FILE_PREFIX) + trace.Trace("cleaning locks:", cmd) + backticks.Run("/bin/sh", "-c", cmd) +} diff --git a/client/go/internal/admin/script-utils/logfmt/cmd.go b/client/go/internal/admin/script-utils/logfmt/cmd.go new file mode 100644 index 00000000000..a8675c37356 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/cmd.go @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-logfmt command +// Author: arnej + +package logfmt + +import ( + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/cli/build" +) + +func NewLogfmtCmd() *cobra.Command { + var ( + curOptions Options = NewOptions() + ) + cmd := &cobra.Command{ + Use: "vespa-logfmt", + Short: "convert vespa.log to human-readable format", + Long: `vespa-logfmt takes input in the internal vespa format +and converts it to something human-readable`, + Version: build.Version, + Run: func(cmd *cobra.Command, args []string) { + RunLogfmt(&curOptions, args) + }, + } + cmd.Flags().VarP(&curOptions.ShowLevels, "level", "l", "turn levels on/off\n") + cmd.Flags().VarP(&curOptions.ShowFields, "show", "s", "turn fields shown on/off\n") + cmd.Flags().VarP(&curOptions.ComponentFilter, "component", "c", "select components by regexp") + cmd.Flags().VarP(&curOptions.MessageFilter, "message", "m", "select messages by regexp") + cmd.Flags().BoolVarP(&curOptions.OnlyInternal, "internal", "i", false, "select only internal components") + cmd.Flags().BoolVar(&curOptions.TruncateService, "truncateservice", false, "truncate service name") + cmd.Flags().BoolVar(&curOptions.TruncateService, "ts", false, "") + cmd.Flags().BoolVarP(&curOptions.FollowTail, "follow", "f", false, "follow logfile with tail -f") + cmd.Flags().BoolVarP(&curOptions.DequoteNewlines, "nldequote", "N", false, "dequote newlines embedded in message") + cmd.Flags().BoolVarP(&curOptions.DequoteNewlines, "dequotenewlines", "n", false, "dequote newlines embedded in message") + cmd.Flags().BoolVarP(&curOptions.TruncateComponent, "truncatecomponent", "t", false, "truncate component name") + cmd.Flags().BoolVar(&curOptions.TruncateComponent, "tc", false, "") + cmd.Flags().StringVarP(&curOptions.OnlyHostname, "host", "H", "", "select only one host") + cmd.Flags().StringVarP(&curOptions.OnlyPid, "pid", "p", "", "select only one process ID") + cmd.Flags().StringVarP(&curOptions.OnlyService, "service", "S", "", "select only one service") + cmd.Flags().VarP(&curOptions.Format, "format", "F", "select logfmt output format, vespa (default), json or raw are supported. The json output format is not stable, and will change in the future.") + cmd.Flags().MarkHidden("tc") + cmd.Flags().MarkHidden("ts") + cmd.Flags().MarkHidden("dequotenewlines") + return cmd +} diff --git a/client/go/internal/admin/script-utils/logfmt/formatflags.go b/client/go/internal/admin/script-utils/logfmt/formatflags.go new file mode 100644 index 00000000000..097746d696f --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/formatflags.go @@ -0,0 +1,41 @@ +package logfmt + +import ( + "fmt" + "strings" +) + +type OutputFormat int + +const ( + FormatVespa OutputFormat = iota //default is vespa + FormatRaw + FormatJSON +) + +func (v *OutputFormat) Type() string { + return "output format" +} + +func (v *OutputFormat) String() string { + flagNames := []string{ + "vespa", + "raw", + "json", + } + return flagNames[*v] +} + +func (v *OutputFormat) Set(val string) error { + switch strings.ToLower(val) { + case "vespa": + *v = FormatVespa + case "raw": + *v = FormatRaw + case "json": + *v = FormatJSON + default: + return fmt.Errorf("'%s' is not a valid format argument", val) + } + return nil +} diff --git a/client/go/internal/admin/script-utils/logfmt/formatflags_test.go b/client/go/internal/admin/script-utils/logfmt/formatflags_test.go new file mode 100644 index 00000000000..53c47d24208 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/formatflags_test.go @@ -0,0 +1,30 @@ +package logfmt + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestOutputFormat(t *testing.T) { + type args struct { + val string + } + tests := []struct { + expected OutputFormat + arg string + wantErr assert.ErrorAssertionFunc + }{ + {FormatVespa, "vespa", assert.NoError}, + {FormatRaw, "raw", assert.NoError}, + {FormatJSON, "json", assert.NoError}, + {-1, "foo", assert.Error}, + } + for _, tt := range tests { + t.Run(tt.arg, func(t *testing.T) { + var v OutputFormat = -1 + tt.wantErr(t, v.Set(tt.arg), fmt.Sprintf("Set(%v)", tt.arg)) + assert.Equal(t, v, tt.expected) + }) + } +} diff --git a/client/go/internal/admin/script-utils/logfmt/handleline.go b/client/go/internal/admin/script-utils/logfmt/handleline.go new file mode 100644 index 00000000000..813ca82acb4 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/handleline.go @@ -0,0 +1,179 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" +) + +type logFields struct { + timestamp string // seconds, optional fractional seconds + host string + pid string // pid, optional tid + service string + component string + level string + messages []string +} + +// handle a line in "vespa.log" format; do filtering and formatting as specified in opts +func handleLine(opts *Options, line string) (string, error) { + fieldStrings := strings.SplitN(line, "\t", 7) + if len(fieldStrings) < 7 { + return "", fmt.Errorf("not enough fields: '%s'", line) + } + fields := logFields{ + timestamp: fieldStrings[0], + host: fieldStrings[1], + pid: fieldStrings[2], + service: fieldStrings[3], + component: fieldStrings[4], + level: fieldStrings[5], + messages: fieldStrings[6:], + } + + if !opts.showLevel(fields.level) { + return "", nil + } + if opts.OnlyHostname != "" && opts.OnlyHostname != fields.host { + return "", nil + } + if opts.OnlyPid != "" && opts.OnlyPid != fields.pid { + return "", nil + } + if opts.OnlyService != "" && opts.OnlyService != fields.service { + return "", nil + } + if opts.OnlyInternal && !isInternal(fields.component) { + return "", nil + } + if opts.ComponentFilter.unmatched(fields.component) { + return "", nil + } + if opts.MessageFilter.unmatched(strings.Join(fields.messages, "\t")) { + return "", nil + } + + switch opts.Format { + case FormatRaw: + return line + "\n", nil + case FormatJSON: + return handleLineJson(opts, &fields) + case FormatVespa: + fallthrough + default: + return handleLineVespa(opts, &fields) + } +} + +func parseTimestamp(timestamp string) (time.Time, error) { + secs, err := strconv.ParseFloat(timestamp, 64) + if err != nil { + return time.Time{}, err + } + nsecs := int64(secs * 1e9) + return time.Unix(0, nsecs), nil +} + +type logFieldsJson struct { + Timestamp string `json:"timestamp"` + Host string `json:"host"` + Pid string `json:"pid"` + Service string `json:"service"` + Component string `json:"component"` + Level string `json:"level"` + Messages []string `json:"messages"` +} + +func handleLineJson(_ *Options, fields *logFields) (string, error) { + timestamp, err := parseTimestamp(fields.timestamp) + if err != nil { + return "", err + } + outputFields := logFieldsJson{ + Timestamp: timestamp.Format(time.RFC3339Nano), + Host: fields.host, + Pid: fields.pid, + Service: fields.service, + Component: fields.component, + Level: fields.level, + Messages: fields.messages, + } + buf := bytes.Buffer{} + if err := json.NewEncoder(&buf).Encode(&outputFields); err != nil { + return "", err + } + return buf.String(), nil +} + +func handleLineVespa(opts *Options, fields *logFields) (string, error) { + var buf strings.Builder + + if opts.showField("fmttime") { + timestamp, err := parseTimestamp(fields.timestamp) + if err != nil { + return "", err + } + if opts.showField("usecs") { + buf.WriteString(timestamp.Format("[2006-01-02 15:04:05.000000] ")) + } else if opts.showField("msecs") { + buf.WriteString(timestamp.Format("[2006-01-02 15:04:05.000] ")) + } else { + buf.WriteString(timestamp.Format("[2006-01-02 15:04:05] ")) + } + } else if opts.showField("time") { + buf.WriteString(fields.timestamp) + buf.WriteString(" ") + } + if opts.showField("host") { + buf.WriteString(fmt.Sprintf("%-8s ", fields.host)) + } + if opts.showField("level") { + buf.WriteString(fmt.Sprintf("%-7s ", strings.ToUpper(fields.level))) + } + if opts.showField("pid") { + // OnlyPid, _, _ := strings.Cut(pidfield, "/") + buf.WriteString(fmt.Sprintf("%6s ", fields.pid)) + } + if opts.showField("service") { + if opts.TruncateService { + buf.WriteString(fmt.Sprintf("%-9.9s ", fields.service)) + } else { + buf.WriteString(fmt.Sprintf("%-16s ", fields.service)) + } + } + if opts.showField("component") { + if opts.TruncateComponent { + buf.WriteString(fmt.Sprintf("%-15.15s ", fields.component)) + } else { + buf.WriteString(fmt.Sprintf("%s\t", fields.component)) + } + } + if opts.showField("message") { + var msgBuf strings.Builder + for idx, message := range fields.messages { + if idx > 0 { + msgBuf.WriteString("\n\t") + } + if opts.DequoteNewlines { + message = strings.ReplaceAll(message, "\\n\\t", "\n\t") + message = strings.ReplaceAll(message, "\\n", "\n\t") + } + msgBuf.WriteString(message) + } + message := msgBuf.String() + if strings.Contains(message, "\n") { + buf.WriteString("\n\t") + } + buf.WriteString(message) + } + buf.WriteString("\n") + return buf.String(), nil +} diff --git a/client/go/internal/admin/script-utils/logfmt/internal.go b/client/go/internal/admin/script-utils/logfmt/internal.go new file mode 100644 index 00000000000..992c537f939 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/internal.go @@ -0,0 +1,106 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" +) + +// is componentName a vespa-internal name? + +func isInternal(componentName string) bool { + cs := strings.Split(componentName, ".") + if len(cs) == 0 || cs[0] != "Container" { + return true + } + if len(cs) < 3 { + return false + } + if cs[1] == "ai" && cs[2] == "vespa" { + return true + } + if cs[1] == "com" && cs[2] == "yahoo" && len(cs) > 3 { + return internalComYahooNames[cs[3]] + } + return false +} + +// a constant: +var internalComYahooNames = map[string]bool{ + "abicheck": true, + "api": true, + "application": true, + "binaryprefix": true, + "clientmetrics": true, + "cloud": true, + "collections": true, + "component": true, + "compress": true, + "concurrent": true, + "configtest": true, + "config": true, + "container": true, + "data": true, + "docprocs": true, + "docproc": true, + "documentapi": true, + "documentmodel": true, + "document": true, + "dummyreceiver": true, + "embedding": true, + "errorhandling": true, + "exception": true, + "feedapi": true, + "feedhandler": true, + "filedistribution": true, + "fs4": true, + "fsa": true, + "geo": true, + "io": true, + "javacc": true, + "jdisc": true, + "jrt": true, + "lang": true, + "language": true, + "logserver": true, + "log": true, + "messagebus": true, + "metrics": true, + "nativec": true, + "net": true, + "osgi": true, + "path": true, + "plugin": true, + "prelude": true, + "processing": true, + "protect": true, + "reflection": true, + "restapi": true, + "schema": true, + "searchdefinition": true, + "searchlib": true, + "search": true, + "security": true, + "slime": true, + "socket": true, + "statistics": true, + "stream": true, + "system": true, + "tensor": true, + "test": true, + "text": true, + "time": true, + "transaction": true, + "vdslib": true, + "vespaclient": true, + "vespafeeder": true, + "vespaget": true, + "vespastat": true, + "vespasummarybenchmark": true, + "vespa": true, + "vespavisit": true, + "vespaxmlparser": true, + "yolean": true, +} diff --git a/client/go/internal/admin/script-utils/logfmt/internal_names.txt b/client/go/internal/admin/script-utils/logfmt/internal_names.txt new file mode 100644 index 00000000000..cc554546fcc --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/internal_names.txt @@ -0,0 +1,782 @@ +sentinel.sentinel.config +searchnode.eval.Test +Container.ai.vespa.client.dsl.Test +Container.ai.vespa.cloud.Test +Container.ai.vespa.embedding.Test +Container.ai.vespa.explicitversion.Test +Container.ai.vespa.explicitversion_dep.Test +Container.ai.vespa.feed.client.Test +Container.ai.vespa.feed.client.impl.Test +Container.ai.vespa.hosted.api.Test +Container.ai.vespa.hosted.cd.Test +Container.ai.vespa.hosted.cd.cloud.impl.Test +Container.ai.vespa.hosted.cd.commons.Test +Container.ai.vespa.hosted.cd.internal.Test +Container.ai.vespa.hosted.client.Test +Container.ai.vespa.hosted.plugin.Test +Container.ai.vespa.http.Test +Container.ai.vespa.intellij.schema.Test +Container.ai.vespa.intellij.schema.findUsages.Test +Container.ai.vespa.intellij.schema.hierarchy.Test +Container.ai.vespa.intellij.schema.lexer.Test +Container.ai.vespa.intellij.schema.model.Test +Container.ai.vespa.intellij.schema.parser.Test +Container.ai.vespa.intellij.schema.psi.Test +Container.ai.vespa.intellij.schema.psi.impl.Test +Container.ai.vespa.intellij.schema.structure.Test +Container.ai.vespa.intellij.schema.utils.Test +Container.ai.vespa.logserver.protocol.Test +Container.ai.vespa.metricsproxy.core.Test +Container.ai.vespa.metricsproxy.http.Test +Container.ai.vespa.metricsproxy.http.application.Test +Container.ai.vespa.metricsproxy.http.metrics.Test +Container.ai.vespa.metricsproxy.http.prometheus.Test +Container.ai.vespa.metricsproxy.http.yamas.Test +Container.ai.vespa.metricsproxy.metric.Test +Container.ai.vespa.metricsproxy.metric.dimensions.Test +Container.ai.vespa.metricsproxy.metric.model.Test +Container.ai.vespa.metricsproxy.metric.model.json.Test +Container.ai.vespa.metricsproxy.metric.model.processing.Test +Container.ai.vespa.metricsproxy.metric.model.prometheus.Test +Container.ai.vespa.metricsproxy.node.Test +Container.ai.vespa.metricsproxy.rpc.Test +Container.ai.vespa.metricsproxy.service.Test +Container.ai.vespa.metricsproxy.telegraf.Test +Container.ai.vespa.modelintegration.evaluator.Test +Container.ai.vespa.models.evaluation.Test +Container.ai.vespa.models.handler.Test +Container.ai.vespa.noversion.Test +Container.ai.vespa.noversion_dep.Test +Container.ai.vespa.rankingexpression.importer.Test +Container.ai.vespa.rankingexpression.importer.configmodelview.Test +Container.ai.vespa.rankingexpression.importer.lightgbm.Test +Container.ai.vespa.rankingexpression.importer.onnx.Test +Container.ai.vespa.rankingexpression.importer.operations.Test +Container.ai.vespa.rankingexpression.importer.tensorflow.Test +Container.ai.vespa.rankingexpression.importer.vespa.Test +Container.ai.vespa.rankingexpression.importer.vespa.parser.Test +Container.ai.vespa.rankingexpression.importer.xgboost.Test +Container.ai.vespa.reindexing.Test +Container.ai.vespa.reindexing.http.Test +Container.ai.vespa.searchlib.searchprotocol.protobuf.Test +Container.ai.vespa.util.http.hc4.Test +Container.ai.vespa.util.http.hc4.retry.Test +Container.ai.vespa.util.http.hc5.Test +Container.ai.vespa.validation.Test +Container.com.yahoo.abicheck.classtree.Test +Container.com.yahoo.abicheck.collector.Test +Container.com.yahoo.abicheck.mojo.Test +Container.com.yahoo.abicheck.setmatcher.Test +Container.com.yahoo.abicheck.signature.Test +Container.com.yahoo.api.annotations.Test +Container.com.yahoo.application.Test +Container.com.yahoo.application.container.Test +Container.com.yahoo.application.container.handler.Test +Container.com.yahoo.application.container.impl.Test +Container.com.yahoo.application.content.Test +Container.com.yahoo.application.preprocessor.Test +Container.com.yahoo.binaryprefix.Test +Container.com.yahoo.clientmetrics.Test +Container.com.yahoo.cloud.config.Test +Container.com.yahoo.collections.Test +Container.com.yahoo.component.Test +Container.com.yahoo.component.annotation.Test +Container.com.yahoo.component.chain.Test +Container.com.yahoo.component.chain.dependencies.Test +Container.com.yahoo.component.chain.dependencies.ordering.Test +Container.com.yahoo.component.chain.model.Test +Container.com.yahoo.component.provider.Test +Container.com.yahoo.compress.Test +Container.com.yahoo.concurrent.Test +Container.com.yahoo.concurrent.classlock.Test +Container.com.yahoo.concurrent.maintenance.Test +Container.com.yahoo.config.Test +Container.com.yahoo.config.application.Test +Container.com.yahoo.config.application.api.Test +Container.com.yahoo.config.application.api.xml.Test +Container.com.yahoo.config.codegen.Test +Container.com.yahoo.config.docproc.Test +Container.com.yahoo.config.ini.Test +Container.com.yahoo.config.model.Test +Container.com.yahoo.config.model.admin.Test +Container.com.yahoo.config.model.api.Test +Container.com.yahoo.config.model.api.container.Test +Container.com.yahoo.config.model.application.Test +Container.com.yahoo.config.model.application.provider.Test +Container.com.yahoo.config.model.builder.xml.Test +Container.com.yahoo.config.model.deploy.Test +Container.com.yahoo.config.model.graph.Test +Container.com.yahoo.config.model.producer.Test +Container.com.yahoo.config.model.provision.Test +Container.com.yahoo.config.model.test.Test +Container.com.yahoo.config.provision.Test +Container.com.yahoo.config.provision.exception.Test +Container.com.yahoo.config.provision.host.Test +Container.com.yahoo.config.provisioning.Test +Container.com.yahoo.config.provision.security.Test +Container.com.yahoo.config.provision.serialization.Test +Container.com.yahoo.config.provision.zone.Test +Container.com.yahoo.config.subscription.Test +Container.com.yahoo.config.subscription.impl.Test +Container.com.yahoo.configtest.Test +Container.com.yahoo.config.text.Test +Container.com.yahoo.container.Test +Container.com.yahoo.container.bundle.Test +Container.com.yahoo.container.core.Test +Container.com.yahoo.container.core.config.Test +Container.com.yahoo.container.core.config.testutil.Test +Container.com.yahoo.container.core.document.Test +Container.com.yahoo.container.core.documentapi.Test +Container.com.yahoo.container.core.http.Test +Container.com.yahoo.container.core.identity.Test +Container.com.yahoo.container.di.Test +Container.com.yahoo.container.di.componentgraph.Test +Container.com.yahoo.container.di.componentgraph.core.Test +Container.com.yahoo.container.di.componentgraph.cycle.Test +Container.com.yahoo.container.di.config.Test +Container.com.yahoo.container.handler.Test +Container.com.yahoo.container.handler.metrics.Test +Container.com.yahoo.container.handler.observability.Test +Container.com.yahoo.container.handler.test.Test +Container.com.yahoo.container.handler.threadpool.Test +Container.com.yahoo.container.http.Test +Container.com.yahoo.container.http.filter.Test +Container.com.yahoo.container.jdisc.Test +Container.com.yahoo.container.jdisc.athenz.Test +Container.com.yahoo.container.jdisc.component.Test +Container.com.yahoo.container.jdisc.config.Test +Container.com.yahoo.container.jdisc.messagebus.Test +Container.com.yahoo.container.jdisc.metric.Test +Container.com.yahoo.container.jdisc.metric.state.Test +Container.com.yahoo.container.jdisc.secretstore.Test +Container.com.yahoo.container.jdisc.state.Test +Container.com.yahoo.container.jdisc.utils.Test +Container.com.yahoo.container.logging.Test +Container.com.yahoo.container.plugin.bundle.Test +Container.com.yahoo.container.plugin.classanalysis.Test +Container.com.yahoo.container.plugin.mojo.Test +Container.com.yahoo.container.plugin.osgi.Test +Container.com.yahoo.container.plugin.util.Test +Container.com.yahoo.container.protect.Test +Container.com.yahoo.container.standalone.Test +Container.com.yahoo.container.usability.Test +Container.com.yahoo.data.Test +Container.com.yahoo.data.access.Test +Container.com.yahoo.data.access.simple.Test +Container.com.yahoo.data.access.slime.Test +Container.com.yahoo.docproc.Test +Container.com.yahoo.docproc.impl.Test +Container.com.yahoo.docproc.jdisc.Test +Container.com.yahoo.docproc.jdisc.messagebus.Test +Container.com.yahoo.docproc.jdisc.metric.Test +Container.com.yahoo.docproc.jdisc.observability.Test +Container.com.yahoo.docproc.proxy.Test +Container.com.yahoo.docprocs.indexing.Test +Container.com.yahoo.document.Test +Container.com.yahoo.document.annotation.Test +Container.com.yahoo.documentapi.Test +Container.com.yahoo.documentapi.local.Test +Container.com.yahoo.documentapi.messagebus.Test +Container.com.yahoo.documentapi.messagebus.protocol.Test +Container.com.yahoo.documentapi.messagebus.systemstate.rule.Test +Container.com.yahoo.documentapi.metrics.Test +Container.com.yahoo.document.config.Test +Container.com.yahoo.document.datatypes.Test +Container.com.yahoo.document.fieldpathupdate.Test +Container.com.yahoo.document.fieldset.Test +Container.com.yahoo.document.idstring.Test +Container.com.yahoo.document.internal.Test +Container.com.yahoo.document.json.Test +Container.com.yahoo.document.json.document.Test +Container.com.yahoo.document.json.readers.Test +Container.com.yahoo.documentmodel.Test +Container.com.yahoo.document.predicate.Test +Container.com.yahoo.document.restapi.resource.Test +Container.com.yahoo.document.select.Test +Container.com.yahoo.document.select.convert.Test +Container.com.yahoo.document.select.parser.Test +Container.com.yahoo.document.select.rule.Test +Container.com.yahoo.document.select.simple.Test +Container.com.yahoo.document.serialization.Test +Container.com.yahoo.document.update.Test +Container.com.yahoo.dummyreceiver.Test +Container.com.yahoo.embedding.Test +Container.com.yahoo.errorhandling.Test +Container.com.yahoo.exception.Test +Container.com.yahoo.feedapi.Test +Container.com.yahoo.feedhandler.Test +Container.com.yahoo.filedistribution.fileacquirer.Test +Container.com.yahoo.fs4.Test +Container.com.yahoo.fsa.Test +Container.com.yahoo.fsa.conceptnet.Test +Container.com.yahoo.fsa.segmenter.Test +Container.com.yahoo.fsa.topicpredictor.Test +Container.com.yahoo.geo.Test +Container.com.yahoo.io.Test +Container.com.yahoo.io.reader.Test +Container.com.yahoo.javacc.Test +Container.com.yahoo.jdisc.Test +Container.com.yahoo.jdisc.application.Test +Container.com.yahoo.jdisc.bundle.Test +Container.com.yahoo.jdisc.bundle.a.Test +Container.com.yahoo.jdisc.bundle.b.Test +Container.com.yahoo.jdisc.bundle.c.Test +Container.com.yahoo.jdisc.bundle.d.Test +Container.com.yahoo.jdisc.bundle.e.Test +Container.com.yahoo.jdisc.bundle.f.Test +Container.com.yahoo.jdisc.bundle.g.Test +Container.com.yahoo.jdisc.bundle.g_act.Test +Container.com.yahoo.jdisc.bundle.h.Test +Container.com.yahoo.jdisc.bundle.i.Test +Container.com.yahoo.jdisc.bundle.j.Test +Container.com.yahoo.jdisc.bundle.k.Test +Container.com.yahoo.jdisc.bundle.l.Test +Container.com.yahoo.jdisc.bundle.m.Test +Container.com.yahoo.jdisc.bundle.my_act.Test +Container.com.yahoo.jdisc.bundle.n.Test +Container.com.yahoo.jdisc.bundle.o.Test +Container.com.yahoo.jdisc.bundle.p.Test +Container.com.yahoo.jdisc.bundle.q.Test +Container.com.yahoo.jdisc.bundle.r.Test +Container.com.yahoo.jdisc.bundle.s.Test +Container.com.yahoo.jdisc.bundle.t.Test +Container.com.yahoo.jdisc.bundle.u.Test +Container.com.yahoo.jdisc.client.Test +Container.com.yahoo.jdisc.cloud.aws.Test +Container.com.yahoo.jdisc.core.Test +Container.com.yahoo.jdisc.handler.Test +Container.com.yahoo.jdisc.http.Test +Container.com.yahoo.jdisc.http.cloud.Test +Container.com.yahoo.jdisc.http.filter.Test +Container.com.yahoo.jdisc.http.filter.chain.Test +Container.com.yahoo.jdisc.http.filter.security.athenz.Test +Container.com.yahoo.jdisc.http.filter.security.base.Test +Container.com.yahoo.jdisc.http.filter.security.cors.Test +Container.com.yahoo.jdisc.http.filter.security.csp.Test +Container.com.yahoo.jdisc.http.filter.security.misc.Test +Container.com.yahoo.jdisc.http.filter.security.rule.Test +Container.com.yahoo.jdisc.http.filter.util.Test +Container.com.yahoo.jdisc.http.server.jetty.Test +Container.com.yahoo.jdisc.http.server.jetty.testutils.Test +Container.com.yahoo.jdisc.http.ssl.impl.Test +Container.com.yahoo.jdisc.metrics.yamasconsumer.cloud.Test +Container.com.yahoo.jdisc.refcount.Test +Container.com.yahoo.jdisc.service.Test +Container.com.yahoo.jdisc.statistics.Test +Container.com.yahoo.jdisc.test.Test +Container.com.yahoo.jrt.Test +Container.com.yahoo.jrt.slobrok.Test +Container.com.yahoo.jrt.slobrok.api.Test +Container.com.yahoo.jrt.slobrok.server.Test +Container.com.yahoo.jrt.tool.Test +Container.com.yahoo.lang.Test +Container.com.yahoo.language.Test +Container.com.yahoo.language.detect.Test +Container.com.yahoo.language.opennlp.Test +Container.com.yahoo.language.process.Test +Container.com.yahoo.language.provider.Test +Container.com.yahoo.language.sentencepiece.Test +Container.com.yahoo.language.simple.Test +Container.com.yahoo.language.simple.kstem.Test +Container.com.yahoo.language.tools.Test +Container.com.yahoo.language.wordpiece.Test +Container.com.yahoo.log.Test +Container.com.yahoo.log.event.Test +Container.com.yahoo.log.impl.Test +Container.com.yahoo.logserver.Test +Container.com.yahoo.logserver.filter.Test +Container.com.yahoo.logserver.handlers.Test +Container.com.yahoo.logserver.handlers.archive.Test +Container.com.yahoo.logserver.handlers.logmetrics.Test +Container.com.yahoo.logserver.testutils.Test +Container.com.yahoo.messagebus.Test +Container.com.yahoo.messagebus.jdisc.Test +Container.com.yahoo.messagebus.jdisc.test.Test +Container.com.yahoo.messagebus.network.Test +Container.com.yahoo.messagebus.network.local.Test +Container.com.yahoo.messagebus.network.rpc.Test +Container.com.yahoo.messagebus.network.rpc.test.Test +Container.com.yahoo.messagebus.routing.Test +Container.com.yahoo.messagebus.routing.test.Test +Container.com.yahoo.messagebus.shared.Test +Container.com.yahoo.messagebus.test.Test +Container.com.yahoo.metrics.Test +Container.com.yahoo.metrics.simple.Test +Container.com.yahoo.metrics.simple.jdisc.Test +Container.com.yahoo.metrics.simple.runtime.Test +Container.com.yahoo.nativec.Test +Container.com.yahoo.net.Test +Container.com.yahoo.osgi.Test +Container.com.yahoo.osgi.annotation.Test +Container.com.yahoo.osgi.provider.model.Test +Container.com.yahoo.path.Test +Container.com.yahoo.plugin.Test +Container.com.yahoo.prelude.Test +Container.com.yahoo.prelude.cluster.Test +Container.com.yahoo.prelude.fastsearch.Test +Container.com.yahoo.prelude.hitfield.Test +Container.com.yahoo.prelude.query.Test +Container.com.yahoo.prelude.query.parser.Test +Container.com.yahoo.prelude.query.textualrepresentation.Test +Container.com.yahoo.prelude.querytransform.Test +Container.com.yahoo.prelude.searcher.Test +Container.com.yahoo.prelude.semantics.Test +Container.com.yahoo.prelude.semantics.benchmark.Test +Container.com.yahoo.prelude.semantics.config.Test +Container.com.yahoo.prelude.semantics.engine.Test +Container.com.yahoo.prelude.semantics.parser.Test +Container.com.yahoo.prelude.semantics.rule.Test +Container.com.yahoo.prelude.statistics.Test +Container.com.yahoo.processing.Test +Container.com.yahoo.processing.execution.Test +Container.com.yahoo.processing.execution.chain.Test +Container.com.yahoo.processing.handler.Test +Container.com.yahoo.processing.impl.Test +Container.com.yahoo.processing.processors.Test +Container.com.yahoo.processing.rendering.Test +Container.com.yahoo.processing.request.Test +Container.com.yahoo.processing.request.properties.Test +Container.com.yahoo.processing.response.Test +Container.com.yahoo.processing.test.Test +Container.com.yahoo.protect.Test +Container.com.yahoo.reflection.Test +Container.com.yahoo.restapi.Test +Container.com.yahoo.schema.Test +Container.com.yahoo.schema.derived.Test +Container.com.yahoo.schema.derived.validation.Test +Container.com.yahoo.schema.document.Test +Container.com.yahoo.schema.document.annotation.Test +Container.com.yahoo.schema.expressiontransforms.Test +Container.com.yahoo.schema.fieldoperation.Test +Container.com.yahoo.schema.parser.Test +Container.com.yahoo.schema.processing.Test +Container.com.yahoo.schema.processing.multifieldresolver.Test +Container.com.yahoo.search.Test +Container.com.yahoo.search.cluster.Test +Container.com.yahoo.search.config.Test +Container.com.yahoo.search.dispatch.Test +Container.com.yahoo.search.dispatch.rpc.Test +Container.com.yahoo.search.dispatch.searchcluster.Test +Container.com.yahoo.search.federation.Test +Container.com.yahoo.search.federation.selection.Test +Container.com.yahoo.search.federation.sourceref.Test +Container.com.yahoo.search.grouping.Test +Container.com.yahoo.search.grouping.request.Test +Container.com.yahoo.search.grouping.request.parser.Test +Container.com.yahoo.search.grouping.result.Test +Container.com.yahoo.search.grouping.vespa.Test +Container.com.yahoo.search.handler.Test +Container.com.yahoo.search.handler.observability.Test +Container.com.yahoo.search.intent.model.Test +Container.com.yahoo.searchlib.Test +Container.com.yahoo.searchlib.aggregation.Test +Container.com.yahoo.searchlib.aggregation.hll.Test +Container.com.yahoo.searchlib.document.Test +Container.com.yahoo.searchlib.expression.Test +Container.com.yahoo.searchlib.gbdt.Test +Container.com.yahoo.searchlib.rankingexpression.Test +Container.com.yahoo.searchlib.rankingexpression.evaluation.Test +Container.com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization.Test +Container.com.yahoo.searchlib.rankingexpression.evaluation.tensoroptimization.Test +Container.com.yahoo.searchlib.rankingexpression.parser.Test +Container.com.yahoo.searchlib.rankingexpression.rule.Test +Container.com.yahoo.searchlib.rankingexpression.transform.Test +Container.com.yahoo.searchlib.ranking.features.Test +Container.com.yahoo.searchlib.ranking.features.fieldmatch.Test +Container.com.yahoo.searchlib.tensor.Test +Container.com.yahoo.searchlib.treenet.Test +Container.com.yahoo.searchlib.treenet.parser.Test +Container.com.yahoo.searchlib.treenet.rule.Test +Container.com.yahoo.search.match.Test +Container.com.yahoo.search.pagetemplates.Test +Container.com.yahoo.search.pagetemplates.config.Test +Container.com.yahoo.search.pagetemplates.engine.Test +Container.com.yahoo.search.pagetemplates.engine.resolvers.Test +Container.com.yahoo.search.pagetemplates.model.Test +Container.com.yahoo.search.pagetemplates.result.Test +Container.com.yahoo.search.predicate.Test +Container.com.yahoo.search.predicate.annotator.Test +Container.com.yahoo.search.predicate.benchmarks.Test +Container.com.yahoo.search.predicate.index.Test +Container.com.yahoo.search.predicate.index.conjunction.Test +Container.com.yahoo.search.predicate.optimization.Test +Container.com.yahoo.search.predicate.serialization.Test +Container.com.yahoo.search.predicate.utils.Test +Container.com.yahoo.search.query.Test +Container.com.yahoo.search.query.context.Test +Container.com.yahoo.search.query.gui.Test +Container.com.yahoo.search.query.parser.Test +Container.com.yahoo.search.query.profile.Test +Container.com.yahoo.search.query.profile.compiled.Test +Container.com.yahoo.search.query.profile.config.Test +Container.com.yahoo.search.query.profile.types.Test +Container.com.yahoo.search.query.properties.Test +Container.com.yahoo.search.query.ranking.Test +Container.com.yahoo.search.query.restapi.Test +Container.com.yahoo.search.query.rewrite.Test +Container.com.yahoo.search.query.rewrite.rewriters.Test +Container.com.yahoo.search.query.textserialize.Test +Container.com.yahoo.search.query.textserialize.item.Test +Container.com.yahoo.search.query.textserialize.parser.Test +Container.com.yahoo.search.query.textserialize.serializer.Test +Container.com.yahoo.search.querytransform.Test +Container.com.yahoo.search.rendering.Test +Container.com.yahoo.search.result.Test +Container.com.yahoo.search.schema.Test +Container.com.yahoo.search.schema.internal.Test +Container.com.yahoo.search.searchchain.Test +Container.com.yahoo.search.searchchain.example.Test +Container.com.yahoo.search.searchchain.model.Test +Container.com.yahoo.search.searchchain.model.federation.Test +Container.com.yahoo.search.searchchain.testutil.Test +Container.com.yahoo.search.searchers.Test +Container.com.yahoo.search.statistics.Test +Container.com.yahoo.search.yql.Test +Container.com.yahoo.security.Test +Container.com.yahoo.security.tls.Test +Container.com.yahoo.slime.Test +Container.com.yahoo.socket.test.Test +Container.com.yahoo.stream.Test +Container.com.yahoo.system.Test +Container.com.yahoo.system.execution.Test +Container.com.yahoo.tensor.Test +Container.com.yahoo.tensor.evaluation.Test +Container.com.yahoo.tensor.functions.Test +Container.com.yahoo.tensor.serialization.Test +Container.com.yahoo.test.Test +Container.com.yahoo.test.json.Test +Container.com.yahoo.text.Test +Container.com.yahoo.text.internal.Test +Container.com.yahoo.text.interpretation.Test +Container.com.yahoo.time.Test +Container.com.yahoo.transaction.Test +Container.com.yahoo.vdslib.Test +Container.com.yahoo.vdslib.distribution.Test +Container.com.yahoo.vdslib.state.Test +Container.com.yahoo.vespa.Test +Container.com.yahoo.vespa.applicationmodel.Test +Container.com.yahoo.vespa.athenz.api.Test +Container.com.yahoo.vespa.athenz.aws.Test +Container.com.yahoo.vespa.athenz.client.Test +Container.com.yahoo.vespa.athenz.client.common.Test +Container.com.yahoo.vespa.athenz.client.common.bindings.Test +Container.com.yahoo.vespa.athenz.client.common.serializers.Test +Container.com.yahoo.vespa.athenz.client.zms.Test +Container.com.yahoo.vespa.athenz.client.zms.bindings.Test +Container.com.yahoo.vespa.athenz.client.zts.Test +Container.com.yahoo.vespa.athenz.client.zts.bindings.Test +Container.com.yahoo.vespa.athenz.client.zts.utils.Test +Container.com.yahoo.vespa.athenz.identity.Test +Container.com.yahoo.vespa.athenz.identityprovider.api.Test +Container.com.yahoo.vespa.athenz.identityprovider.api.bindings.Test +Container.com.yahoo.vespa.athenz.identityprovider.client.Test +Container.com.yahoo.vespa.athenz.tls.Test +Container.com.yahoo.vespa.athenz.utils.Test +Container.com.yahoo.vespa.athenz.zpe.Test +Container.com.yahoo.vespaclient.Test +Container.com.yahoo.vespa.clustercontroller.apps.clustercontroller.Test +Container.com.yahoo.vespa.clustercontroller.apputil.communication.http.Test +Container.com.yahoo.vespa.clustercontroller.core.Test +Container.com.yahoo.vespa.clustercontroller.core.database.Test +Container.com.yahoo.vespa.clustercontroller.core.hostinfo.Test +Container.com.yahoo.vespa.clustercontroller.core.listeners.Test +Container.com.yahoo.vespa.clustercontroller.core.restapiv2.Test +Container.com.yahoo.vespa.clustercontroller.core.restapiv2.requests.Test +Container.com.yahoo.vespa.clustercontroller.core.rpc.Test +Container.com.yahoo.vespa.clustercontroller.core.status.Test +Container.com.yahoo.vespa.clustercontroller.core.status.statuspage.Test +Container.com.yahoo.vespa.clustercontroller.utils.communication.async.Test +Container.com.yahoo.vespa.clustercontroller.utils.communication.http.Test +Container.com.yahoo.vespa.clustercontroller.utils.communication.http.writer.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.response.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.server.Test +Container.com.yahoo.vespa.clustercontroller.utils.util.Test +Container.com.yahoo.vespa.config.Test +Container.com.yahoo.vespa.config.benchmark.Test +Container.com.yahoo.vespa.config.buildergen.Test +Container.com.yahoo.vespa.config.content.Test +Container.com.yahoo.vespa.config.content.core.Test +Container.com.yahoo.vespa.config.content.reindexing.Test +Container.com.yahoo.vespa.config.core.Test +Container.com.yahoo.vespa.configdefinition.Test +Container.com.yahoo.vespa.configmodel.producers.Test +Container.com.yahoo.vespa.config.parser.Test +Container.com.yahoo.vespa.config.protocol.Test +Container.com.yahoo.vespa.config.proxy.Test +Container.com.yahoo.vespa.config.proxy.filedistribution.Test +Container.com.yahoo.vespa.config.search.Test +Container.com.yahoo.vespa.config.search.core.Test +Container.com.yahoo.vespa.config.server.Test +Container.com.yahoo.vespa.config.server.application.Test +Container.com.yahoo.vespa.config.server.configchange.Test +Container.com.yahoo.vespa.config.server.deploy.Test +Container.com.yahoo.vespa.config.server.filedistribution.Test +Container.com.yahoo.vespa.configserver.flags.Test +Container.com.yahoo.vespa.configserver.flags.db.Test +Container.com.yahoo.vespa.configserver.flags.http.Test +Container.com.yahoo.vespa.config.server.host.Test +Container.com.yahoo.vespa.config.server.http.Test +Container.com.yahoo.vespa.config.server.http.status.Test +Container.com.yahoo.vespa.config.server.http.v1.Test +Container.com.yahoo.vespa.config.server.http.v2.Test +Container.com.yahoo.vespa.config.server.http.v2.request.Test +Container.com.yahoo.vespa.config.server.http.v2.response.Test +Container.com.yahoo.vespa.config.server.maintenance.Test +Container.com.yahoo.vespa.config.server.metrics.Test +Container.com.yahoo.vespa.config.server.model.Test +Container.com.yahoo.vespa.config.server.modelfactory.Test +Container.com.yahoo.vespa.config.server.monitoring.Test +Container.com.yahoo.vespa.config.server.provision.Test +Container.com.yahoo.vespa.config.server.rpc.Test +Container.com.yahoo.vespa.config.server.rpc.security.Test +Container.com.yahoo.vespa.config.server.session.Test +Container.com.yahoo.vespa.config.server.tenant.Test +Container.com.yahoo.vespa.config.server.version.Test +Container.com.yahoo.vespa.config.server.zookeeper.Test +Container.com.yahoo.vespa.config.storage.Test +Container.com.yahoo.vespa.config.util.Test +Container.com.yahoo.vespa.curator.Test +Container.com.yahoo.vespa.curator.api.Test +Container.com.yahoo.vespa.curator.mock.Test +Container.com.yahoo.vespa.curator.recipes.Test +Container.com.yahoo.vespa.curator.stats.Test +Container.com.yahoo.vespa.curator.transaction.Test +Container.com.yahoo.vespa.defaults.Test +Container.com.yahoo.vespa.document.Test +Container.com.yahoo.vespa.document.dom.Test +Container.com.yahoo.vespa.documentmodel.Test +Container.com.yahoo.vespafeeder.Test +Container.com.yahoo.vespa.feed.perf.Test +Container.com.yahoo.vespa.filedistribution.Test +Container.com.yahoo.vespa.filedistribution.status.Test +Container.com.yahoo.vespa.flags.Test +Container.com.yahoo.vespa.flags.custom.Test +Container.com.yahoo.vespa.flags.file.Test +Container.com.yahoo.vespa.flags.json.Test +Container.com.yahoo.vespa.flags.json.wire.Test +Container.com.yahoo.vespaget.Test +Container.com.yahoo.vespa.hadoop.Test +Container.com.yahoo.vespa.hadoop.mapreduce.Test +Container.com.yahoo.vespa.hadoop.mapreduce.util.Test +Container.com.yahoo.vespa.hadoop.pig.Test +Container.com.yahoo.vespa.hosted.athenz.instanceproviderservice.Test +Container.com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.Test +Container.com.yahoo.vespa.hosted.ca.Test +Container.com.yahoo.vespa.hosted.ca.instance.Test +Container.com.yahoo.vespa.hosted.ca.restapi.Test +Container.com.yahoo.vespa.hosted.controller.Test +Container.com.yahoo.vespa.hosted.controller.api.application.v4.Test +Container.com.yahoo.vespa.hosted.controller.api.application.v4.model.Test +Container.com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.Test +Container.com.yahoo.vespa.hosted.controller.api.configserver.Test +Container.com.yahoo.vespa.hosted.controller.api.identifiers.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.archive.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.artifact.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.athenz.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.aws.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.billing.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.certificates.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.configserver.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.deployment.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.dns.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.entity.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.horizon.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.jira.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.maven.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.noderepository.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.organization.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.repair.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.resource.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.routing.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.secrets.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.stubs.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.user.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.vcmr.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.zone.Test +Container.com.yahoo.vespa.hosted.controller.api.role.Test +Container.com.yahoo.vespa.hosted.controller.api.systemflags.v1.Test +Container.com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.Test +Container.com.yahoo.vespa.hosted.controller.application.Test +Container.com.yahoo.vespa.hosted.controller.application.pkg.Test +Container.com.yahoo.vespa.hosted.controller.archive.Test +Container.com.yahoo.vespa.hosted.controller.athenz.Test +Container.com.yahoo.vespa.hosted.controller.athenz.config.Test +Container.com.yahoo.vespa.hosted.controller.athenz.impl.Test +Container.com.yahoo.vespa.hosted.controller.auditlog.Test +Container.com.yahoo.vespa.hosted.controller.certificate.Test +Container.com.yahoo.vespa.hosted.controller.concurrent.Test +Container.com.yahoo.vespa.hosted.controller.deployment.Test +Container.com.yahoo.vespa.hosted.controller.dns.Test +Container.com.yahoo.vespa.hosted.controller.maintenance.Test +Container.com.yahoo.vespa.hosted.controller.metric.Test +Container.com.yahoo.vespa.hosted.controller.notification.Test +Container.com.yahoo.vespa.hosted.controller.persistence.Test +Container.com.yahoo.vespa.hosted.controller.proxy.Test +Container.com.yahoo.vespa.hosted.controller.restapi.application.Test +Container.com.yahoo.vespa.hosted.controller.restapi.athenz.Test +Container.com.yahoo.vespa.hosted.controller.restapi.billing.Test +Container.com.yahoo.vespa.hosted.controller.restapi.changemanagement.Test +Container.com.yahoo.vespa.hosted.controller.restapi.configserver.Test +Container.com.yahoo.vespa.hosted.controller.restapi.controller.Test +Container.com.yahoo.vespa.hosted.controller.restapi.deployment.Test +Container.com.yahoo.vespa.hosted.controller.restapi.filter.Test +Container.com.yahoo.vespa.hosted.controller.restapi.flags.Test +Container.com.yahoo.vespa.hosted.controller.restapi.horizon.Test +Container.com.yahoo.vespa.hosted.controller.restapi.os.Test +Container.com.yahoo.vespa.hosted.controller.restapi.routing.Test +Container.com.yahoo.vespa.hosted.controller.restapi.systemflags.Test +Container.com.yahoo.vespa.hosted.controller.restapi.user.Test +Container.com.yahoo.vespa.hosted.controller.restapi.zone.v1.Test +Container.com.yahoo.vespa.hosted.controller.restapi.zone.v2.Test +Container.com.yahoo.vespa.hosted.controller.routing.Test +Container.com.yahoo.vespa.hosted.controller.routing.context.Test +Container.com.yahoo.vespa.hosted.controller.routing.rotation.Test +Container.com.yahoo.vespa.hosted.controller.security.Test +Container.com.yahoo.vespa.hosted.controller.support.access.Test +Container.com.yahoo.vespa.hosted.controller.tenant.Test +Container.com.yahoo.vespa.hosted.controller.tls.Test +Container.com.yahoo.vespa.hosted.controller.versions.Test +Container.com.yahoo.vespa.hosted.node.admin.Test +Container.com.yahoo.vespa.hosted.node.admin.component.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.flags.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.state.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.Test +Container.com.yahoo.vespa.hosted.node.admin.container.Test +Container.com.yahoo.vespa.hosted.node.admin.container.image.Test +Container.com.yahoo.vespa.hosted.node.admin.container.metrics.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.acl.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.coredump.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.disk.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.identity.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.sync.Test +Container.com.yahoo.vespa.hosted.node.admin.nodeadmin.Test +Container.com.yahoo.vespa.hosted.node.admin.nodeagent.Test +Container.com.yahoo.vespa.hosted.node.admin.provider.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.editor.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.file.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.fs.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.network.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.process.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.systemd.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.template.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.text.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.yum.Test +Container.com.yahoo.vespa.hosted.provision.Test +Container.com.yahoo.vespa.hosted.provision.applications.Test +Container.com.yahoo.vespa.hosted.provision.autoscale.Test +Container.com.yahoo.vespa.hosted.provision.lb.Test +Container.com.yahoo.vespa.hosted.provision.maintenance.Test +Container.com.yahoo.vespa.hosted.provision.node.Test +Container.com.yahoo.vespa.hosted.provision.node.filter.Test +Container.com.yahoo.vespa.hosted.provision.os.Test +Container.com.yahoo.vespa.hosted.provision.persistence.Test +Container.com.yahoo.vespa.hosted.provision.provisioning.Test +Container.com.yahoo.vespa.hosted.provision.restapi.Test +Container.com.yahoo.vespa.hosted.provision.testutils.Test +Container.com.yahoo.vespa.hosted.routing.Test +Container.com.yahoo.vespa.hosted.routing.nginx.Test +Container.com.yahoo.vespa.hosted.routing.restapi.Test +Container.com.yahoo.vespa.hosted.routing.status.Test +Container.com.yahoo.vespa.hosted.testrunner.Test +Container.com.yahoo.vespa.http.server.Test +Container.com.yahoo.vespa.http.server.util.Test +Container.com.yahoo.vespa.indexinglanguage.Test +Container.com.yahoo.vespa.indexinglanguage.expressions.Test +Container.com.yahoo.vespa.indexinglanguage.linguistics.Test +Container.com.yahoo.vespa.indexinglanguage.parser.Test +Container.com.yahoo.vespa.indexinglanguage.predicate.Test +Container.com.yahoo.vespa.jaxrs.annotation.Test +Container.com.yahoo.vespa.maven.plugin.enforcer.Test +Container.com.yahoo.vespa.model.Test +Container.com.yahoo.vespa.model.admin.Test +Container.com.yahoo.vespa.model.admin.clustercontroller.Test +Container.com.yahoo.vespa.model.admin.metricsproxy.Test +Container.com.yahoo.vespa.model.admin.monitoring.Test +Container.com.yahoo.vespa.model.admin.monitoring.builder.Test +Container.com.yahoo.vespa.model.admin.monitoring.builder.xml.Test +Container.com.yahoo.vespa.model.application.validation.Test +Container.com.yahoo.vespa.model.application.validation.change.Test +Container.com.yahoo.vespa.model.application.validation.change.search.Test +Container.com.yahoo.vespa.model.application.validation.first.Test +Container.com.yahoo.vespa.model.builder.Test +Container.com.yahoo.vespa.model.builder.xml.dom.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.docproc.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.processing.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.search.Test +Container.com.yahoo.vespa.model.clients.Test +Container.com.yahoo.vespa.model.container.Test +Container.com.yahoo.vespa.model.container.component.Test +Container.com.yahoo.vespa.model.container.component.chain.Test +Container.com.yahoo.vespa.model.container.configserver.Test +Container.com.yahoo.vespa.model.container.configserver.option.Test +Container.com.yahoo.vespa.model.container.docproc.Test +Container.com.yahoo.vespa.model.container.docproc.model.Test +Container.com.yahoo.vespa.model.container.http.Test +Container.com.yahoo.vespa.model.container.http.ssl.Test +Container.com.yahoo.vespa.model.container.http.xml.Test +Container.com.yahoo.vespa.model.container.ml.Test +Container.com.yahoo.vespa.model.container.processing.Test +Container.com.yahoo.vespa.model.container.search.Test +Container.com.yahoo.vespa.model.container.search.searchchain.Test +Container.com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains.Test +Container.com.yahoo.vespa.model.container.xml.Test +Container.com.yahoo.vespa.model.container.xml.document.Test +Container.com.yahoo.vespa.model.container.xml.embedder.Test +Container.com.yahoo.vespa.model.content.Test +Container.com.yahoo.vespa.model.content.cluster.Test +Container.com.yahoo.vespa.model.content.engines.Test +Container.com.yahoo.vespa.model.content.storagecluster.Test +Container.com.yahoo.vespa.model.filedistribution.Test +Container.com.yahoo.vespa.model.ml.Test +Container.com.yahoo.vespa.model.routing.Test +Container.com.yahoo.vespa.model.search.Test +Container.com.yahoo.vespa.model.utils.Test +Container.com.yahoo.vespa.model.utils.internal.Test +Container.com.yahoo.vespa.objects.Test +Container.com.yahoo.vespa.orchestrator.Test +Container.com.yahoo.vespa.orchestrator.config.Test +Container.com.yahoo.vespa.orchestrator.controller.Test +Container.com.yahoo.vespa.orchestrator.model.Test +Container.com.yahoo.vespa.orchestrator.policy.Test +Container.com.yahoo.vespa.orchestrator.resources.Test +Container.com.yahoo.vespa.orchestrator.restapi.wire.Test +Container.com.yahoo.vespa.orchestrator.status.Test +Container.com.yahoo.vespa.orchestrator.status.json.Test +Container.com.yahoo.vespa.security.tool.securityenv.Test +Container.com.yahoo.vespa.service.duper.Test +Container.com.yahoo.vespa.service.executor.Test +Container.com.yahoo.vespa.service.health.Test +Container.com.yahoo.vespa.service.manager.Test +Container.com.yahoo.vespa.service.model.Test +Container.com.yahoo.vespa.service.monitor.Test +Container.com.yahoo.vespa.service.slobrok.Test +Container.com.yahoo.vespastat.Test +Container.com.yahoo.vespa.streamingvisitors.Test +Container.com.yahoo.vespa.streamingvisitors.tracing.Test +Container.com.yahoo.vespasummarybenchmark.Test +Container.com.yahoo.vespa.test.file.Test +Container.com.yahoo.vespa.testrunner.Test +Container.com.yahoo.vespavisit.Test +Container.com.yahoo.vespaxmlparser.Test +Container.com.yahoo.vespa.zookeeper.Test +Container.com.yahoo.vespa.zookeeper.cli.Test +Container.com.yahoo.vespa.zookeeper.client.Test +Container.com.yahoo.yolean.Test +Container.com.yahoo.yolean.chain.Test +Container.com.yahoo.yolean.concurrent.Test +Container.com.yahoo.yolean.function.Test +Container.com.yahoo.yolean.system.Test +Container.com.yahoo.yolean.trace.Test diff --git a/client/go/internal/admin/script-utils/logfmt/internal_notnames.txt b/client/go/internal/admin/script-utils/logfmt/internal_notnames.txt new file mode 100644 index 00000000000..49543758ac3 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/internal_notnames.txt @@ -0,0 +1,8 @@ +Container.ai.onnx.Test +Container.com.fasterxml.jackson.jaxrs.json.Test +Container.com.google.Test +Container.com.yahooapis.foo.Test +Container.com.yahoo.newssearch.Test +Container.com.yahoo.Test +Container.com.yahoo.testing.Test +Container.org.apache.http.Test diff --git a/client/go/internal/admin/script-utils/logfmt/internal_test.go b/client/go/internal/admin/script-utils/logfmt/internal_test.go new file mode 100644 index 00000000000..9b6b0f8404c --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/internal_test.go @@ -0,0 +1,34 @@ +package logfmt + +import ( + "bufio" + "os" + "testing" +) + +// tests: func isInternal(componentName string) bool + +func TestIsInternal(t *testing.T) { + f, err := os.Open("internal_names.txt") + if err != nil { + t.Fatal("could not read test data") + } + defer f.Close() + for input := bufio.NewScanner(f); input.Scan(); { + if name := input.Text(); !isInternal(name) { + t.Logf("name '%s' should be internal but was not recognized", name) + t.Fail() + } + } + f, err = os.Open("internal_notnames.txt") + if err != nil { + t.Fatal("could not read test data") + } + defer f.Close() + for input := bufio.NewScanner(f); input.Scan(); { + if name := input.Text(); isInternal(name) { + t.Logf("name '%s' should not be internal but was recognized", name) + t.Fail() + } + } +} diff --git a/client/go/internal/admin/script-utils/logfmt/levelflags.go b/client/go/internal/admin/script-utils/logfmt/levelflags.go new file mode 100644 index 00000000000..4e6c1284753 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/levelflags.go @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" +) + +// handle CLI flags for log level filtering + +type flagValueForLevel struct { + levels map[string]bool + changed bool +} + +func defaultLevelFlags() map[string]bool { + return map[string]bool{ + "fatal": true, + "error": true, + "warning": true, + "info": true, + "config": false, + "event": false, + "debug": false, + "spam": false, + } +} + +func (v *flagValueForLevel) Type() string { + return "level flags" +} + +func (v *flagValueForLevel) String() string { + var buf strings.Builder + flagNames := []string{ + "fatal", + "error", + "warning", + "info", + "config", + "event", + "debug", + "spam", + } + for _, flag := range flagNames { + if v.levels[flag] { + buf.WriteString(" +") + } else { + buf.WriteString(" -") + } + buf.WriteString(flag) + } + return buf.String() +} + +func (v *flagValueForLevel) flags() map[string]bool { + return v.levels +} + +func (v *flagValueForLevel) name() string { + return "level" +} + +func (v *flagValueForLevel) unchanged() bool { + return !v.changed +} + +func (v *flagValueForLevel) Set(val string) error { + return applyPlusMinus(val, v) +} diff --git a/client/go/internal/admin/script-utils/logfmt/levelflags_test.go b/client/go/internal/admin/script-utils/logfmt/levelflags_test.go new file mode 100644 index 00000000000..186ea2d96b0 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/levelflags_test.go @@ -0,0 +1,74 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" + "testing" +) + +func TestLevelFlags(t *testing.T) { + none := " -fatal -error -warning -info -config -event -debug -spam" + + var flag flagValueForLevel + if flag.String() != none { + t.Logf("unset flag displays as '%s', expected '%s'", flag.String(), none) + t.Fail() + } + if flag.Type() != "level flags" { + t.Logf("flag type was '%s'", flag.Type()) + t.Fail() + } + check := func(expected string, texts ...string) { + var target flagValueForLevel + // target.levels = defaultLevelFlags() + target.levels = defaultLevelFlags() + for _, text := range texts { + err := target.Set(text) + if err != nil { + t.Fatalf("unexpected error with level flags Set('%s'): %v", text, err) + } + } + got := target.String() + if got != expected { + t.Logf("expected flags [%s] but got: [%s]", expected, got) + t.Fail() + } + } + check(" +fatal +error +warning +info -config -event -debug -spam") + check(" -fatal -error -warning -info -config -event -debug -spam", "-all") + check(" +fatal +error +warning +info +config +event +debug +spam", "all") + check(" +fatal +error +warning +info +config +event +debug +spam", "+all") + check(" -fatal -error -warning -info -config -event +debug -spam", "debug") + check(" +fatal +error +warning +info +config +event +debug -spam", "all-spam") + check(" +fatal +error +warning +info +config +event +debug -spam", "all", "-spam") + check(" +fatal +error +warning -info -config +event -debug -spam", "+event", "-info") + check(" +fatal +error -warning -info -config +event -debug -spam", "+event,-info,-warning,config") + check(" +fatal +error -warning -info +config +event -debug -spam", "+event,-info,-warning,+config") + check(" +fatal +error -warning -info +config +event -debug -spam", "+event,-info", "-warning,+config") + check(" -fatal -error -warning -info +config -event -debug -spam", "+event", "-info", "-warning", "config") + check = func(expectErr string, texts ...string) { + var target flagValueForLevel + target.levels = defaultLevelFlags() + for _, text := range texts { + err := target.Set(text) + if err != nil { + if err.Error() == expectErr { + return + } + t.Fatalf("expected error [%s] with level flags Set('%s'), but got [%v]", expectErr, text, err) + } + } + t.Logf("Did not get expected error '%s' from %s", expectErr, strings.Join(texts, ",")) + t.Fail() + } + check("not a valid level flag: 'foo'", "foo") + check("not a valid level flag: 'foo'", "event,foo,config") + check("not a valid level flag: 'foo'", "-event,-foo,-config") + check("not a valid level flag: 'foo'", "+event,+foo,+config") + check("not a valid level flag: 'foo'", "event", "foo", "config") + check("not a valid level flag: 'foo'", "-event", "-foo", "-config") + check("not a valid level flag: 'foo'", "+event", "+foo", "+config") +} diff --git a/client/go/internal/admin/script-utils/logfmt/options.go b/client/go/internal/admin/script-utils/logfmt/options.go new file mode 100644 index 00000000000..864868d4ce5 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/options.go @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "fmt" + "os" +) + +// options designed for compatibility with perl version of vespa-logfmt + +type Options struct { + ShowFields flagValueForShow + ShowLevels flagValueForLevel + OnlyHostname string + OnlyPid string + OnlyService string + OnlyInternal bool + FollowTail bool + DequoteNewlines bool + TruncateService bool + TruncateComponent bool + ComponentFilter regexFlag + MessageFilter regexFlag + Format OutputFormat +} + +func NewOptions() (ret Options) { + ret.ShowLevels.levels = defaultLevelFlags() + ret.ShowFields.shown = defaultShowFlags() + return +} + +func (o *Options) showField(field string) bool { + return o.ShowFields.shown[field] +} + +func (o *Options) showLevel(level string) bool { + rv, ok := o.ShowLevels.levels[level] + if !ok { + o.ShowLevels.levels[level] = true + fmt.Fprintf(os.Stderr, "Warnings: unknown level '%s' in input\n", level) + return true + } + return rv +} diff --git a/client/go/internal/admin/script-utils/logfmt/plusminusflag.go b/client/go/internal/admin/script-utils/logfmt/plusminusflag.go new file mode 100644 index 00000000000..1768cf0e7be --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/plusminusflag.go @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "fmt" + "strings" +) + +// common code for showFlags and levelFlags + +type plusMinusFlag interface { + flags() map[string]bool + name() string + unchanged() bool +} + +func trimPrefix(value, prefix string) (newValue string, hadPrefix bool) { + hadPrefix = strings.HasPrefix(value, prefix) + if hadPrefix { + newValue = strings.TrimPrefix(value, prefix) + } else { + newValue = value + } + return +} + +func applyPlusMinus(val string, target plusMinusFlag) error { + minus := strings.HasPrefix(val, "-") + plus := strings.HasPrefix(val, "+") + val = strings.ReplaceAll(val, "-", ",-") + val = strings.ReplaceAll(val, "+", ",+") + if target.unchanged() { + // user wants to reset flags? + if minus == false && plus == false { + for k, _ := range target.flags() { + target.flags()[k] = false + } + } + } + changeTo := !minus + for _, k := range strings.Split(val, ",") { + if suppress, minus := trimPrefix(k, "-"); minus { + k = suppress + changeTo = false + } + if surface, plus := trimPrefix(k, "+"); plus { + k = surface + changeTo = true + } + if k == "" { + continue + } + if k == "all" { + for k, _ := range target.flags() { + target.flags()[k] = changeTo + } + } else if _, ok := target.flags()[k]; !ok { + return fmt.Errorf("not a valid %s flag: '%s'", target.name(), k) + } else { + target.flags()[k] = changeTo + } + } + return nil +} diff --git a/client/go/internal/admin/script-utils/logfmt/regexflag.go b/client/go/internal/admin/script-utils/logfmt/regexflag.go new file mode 100644 index 00000000000..8f7d2a91373 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/regexflag.go @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "regexp" +) + +// optional regular expression filter, as a CLI flag + +type regexFlag struct { + regex *regexp.Regexp +} + +func (re regexFlag) unmatched(s string) bool { + if re.regex == nil { + return false + } + return re.regex.FindStringIndex(s) == nil +} + +func (v *regexFlag) Type() string { + return "regular expression" +} + +func (v *regexFlag) String() string { + if v.regex == nil { + return "<none>" + } + return v.regex.String() +} + +func (v *regexFlag) Set(val string) (r error) { + v.regex, r = regexp.Compile(val) + return +} diff --git a/client/go/internal/admin/script-utils/logfmt/regexflag_test.go b/client/go/internal/admin/script-utils/logfmt/regexflag_test.go new file mode 100644 index 00000000000..489439863a2 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/regexflag_test.go @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func assertMatch(t *testing.T, re string, flag *regexFlag, texts ...string) { + t.Helper() + err := flag.Set(re) + require.Nil(t, err, "unexpected error with flag.Set('%s'): %v", re, err) + assert.Equal(t, re, flag.String(), "set flag displays as '%s', expected '%s'", flag.String(), re) + for _, text := range texts { + assert.False(t, flag.unmatched(text), "flag '%s' claims a non-match for '%s'", flag.String(), text) + } +} + +func assertUnmatch(t *testing.T, re string, flag *regexFlag, texts ...string) { + t.Helper() + err := flag.Set(re) + require.Nil(t, err, "unexpected error with flag.Set('%s'): %v", re, err) + assert.Equal(t, re, flag.String()) + for _, text := range texts { + assert.True(t, flag.unmatched(text), "flag '%s' should claim a non-match for '%s'", flag.String(), text) + } +} + +func TestRegexFlag(t *testing.T) { + var flag regexFlag + assert.Equal(t, "<none>", flag.String()) + assert.Equal(t, "regular expression", flag.Type()) + assert.False(t, flag.unmatched("foobar"), "unset flag claims a non-match") + assert.EqualError(t, flag.Set("*"), "error parsing regexp: missing argument to repetition operator: `*`") + assertMatch(t, "foo.*bar", new(regexFlag), "foobar", "foo bar", "x foobar y", "xfoobary", "xfooybarz") + assertUnmatch(t, "foo.*bar", new(regexFlag), "Foobar", "foo Bar", "fxoobar", "whatever") +} diff --git a/client/go/internal/admin/script-utils/logfmt/runlogfmt.go b/client/go/internal/admin/script-utils/logfmt/runlogfmt.go new file mode 100644 index 00000000000..6557461598e --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/runlogfmt.go @@ -0,0 +1,85 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "bufio" + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func inputIsPipe() bool { + fi, err := os.Stdin.Stat() + if err != nil { + return false + } + if fi.Mode()&os.ModeNamedPipe == 0 { + return false + } else { + return true + } +} + +// main entry point for vespa-logfmt + +func RunLogfmt(opts *Options, args []string) { + if len(args) == 0 { + if !inputIsPipe() { + args = append(args, vespa.FindHome()+"/logs/vespa/vespa.log") + } else { + formatFile(opts, os.Stdin) + } + } + if opts.FollowTail { + if len(args) != 1 { + fmt.Fprintf(os.Stderr, "Must have exact 1 file for 'follow' option, got %d\n", len(args)) + return + } + if err := tailFile(opts, args[0]); err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + return + } + for _, arg := range args { + file, err := os.Open(arg) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot open '%s': %v\n", arg, err) + } else { + formatFile(opts, file) + file.Close() + } + } +} + +func formatLine(opts *Options, line string) { + output, err := handleLine(opts, line) + if err != nil { + fmt.Fprintln(os.Stderr, "bad log line:", err) + } else { + os.Stdout.WriteString(output) + } +} + +func tailFile(opts *Options, fn string) error { + tailed, err := FollowFile(fn) + if err != nil { + return err + } + for line := range tailed.Lines() { + formatLine(opts, line.Text) + } + return nil +} + +func formatFile(opts *Options, arg *os.File) { + input := bufio.NewScanner(arg) + input.Buffer(make([]byte, 64*1024), 4*1024*1024) + for input.Scan() { + formatLine(opts, input.Text()) + } +} diff --git a/client/go/internal/admin/script-utils/logfmt/showflags.go b/client/go/internal/admin/script-utils/logfmt/showflags.go new file mode 100644 index 00000000000..b69860e0312 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/showflags.go @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" +) + +// handle CLI flags for which fields to show when formatting a line + +type flagValueForShow struct { + shown map[string]bool + changed bool +} + +func defaultShowFlags() map[string]bool { + return map[string]bool{ + "time": true, + "fmttime": true, + "msecs": true, + "usecs": false, + "host": false, + "level": true, + "pid": false, + "service": true, + "component": true, + "message": true, + } +} + +func (v *flagValueForShow) Type() string { + return "show flags" +} + +func (v *flagValueForShow) String() string { + var buf strings.Builder + flagNames := []string{ + "time", + "fmttime", + "msecs", + "usecs", + "host", + "level", + "pid", + "service", + "component", + "message", + } + for _, flag := range flagNames { + if v.shown[flag] { + buf.WriteString(" +") + } else { + buf.WriteString(" -") + } + buf.WriteString(flag) + } + return buf.String() +} + +func (v *flagValueForShow) flags() map[string]bool { + return v.shown +} + +func (v *flagValueForShow) name() string { + return "show" +} + +func (v *flagValueForShow) unchanged() bool { + return !v.changed +} + +func (v *flagValueForShow) Set(val string) error { + return applyPlusMinus(val, v) +} diff --git a/client/go/internal/admin/script-utils/logfmt/showflags_test.go b/client/go/internal/admin/script-utils/logfmt/showflags_test.go new file mode 100644 index 00000000000..d1b66118afd --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/showflags_test.go @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "strings" + "testing" +) + +func TestShowFlags(t *testing.T) { + none := " -time -fmttime -msecs -usecs -host -level -pid -service -component -message" + var flag flagValueForShow + assert.Equal(t, none, flag.String(), "unset flag displays as '%s', expected '%s'", flag.String(), none) + assert.Equal(t, "show flags", flag.Type()) + check := func(expected string, texts ...string) { + var target flagValueForShow + // target.levels = defaultLevelFlags() + target.shown = defaultShowFlags() + for _, text := range texts { + err := target.Set(text) + require.Nil(t, err, "unexpected error with show flags Set('%s'): %v", text, err) + } + assert.Equal(t, expected, target.String()) + } + check(" +time +fmttime +msecs -usecs -host +level -pid +service +component +message") + check(" -time -fmttime -msecs -usecs -host -level -pid -service -component -message", "-all") + check(" +time +fmttime +msecs +usecs +host +level +pid +service +component +message", "all") + check(" +time +fmttime +msecs +usecs +host +level +pid +service +component +message", "+all") + check(" -time -fmttime -msecs -usecs -host -level +pid -service -component -message", "pid") + check(" +time +fmttime +msecs +usecs -host +level +pid +service +component +message", "all-host") + check(" +time +fmttime +msecs +usecs -host +level +pid +service +component +message", "all", "-host") + check(" +time +fmttime -msecs -usecs -host +level +pid +service +component +message", "+pid", "-msecs") + check(" +time -fmttime +msecs -usecs +host +level -pid -service +component +message", "+host,-fmttime,-service,pid") + check(" +time -fmttime +msecs -usecs +host +level +pid -service +component +message", "+host,-fmttime,-service,+pid") + check(" +time -fmttime +msecs -usecs +host +level +pid -service +component +message", "+host,-fmttime", "-service,+pid") + check(" -time -fmttime -msecs -usecs -host -level +pid -service -component -message", "+host", "-fmttime", "-service", "pid") + check = func(expectErr string, texts ...string) { + var target flagValueForShow + target.shown = defaultShowFlags() + for _, text := range texts { + err := target.Set(text) + if err != nil { + require.Equal(t, expectErr, err.Error()) + return + } + } + t.Logf("Did not get expected error [%s] from %s", expectErr, strings.Join(texts, " ")) + t.Fail() + } + check("not a valid show flag: 'foo'", "foo") + check("not a valid show flag: 'foo'", "level,foo,message") + check("not a valid show flag: 'foo'", "-level,-foo,-message") + check("not a valid show flag: 'foo'", "+level,+foo,+message") + check("not a valid show flag: 'foo'", "level", "foo", "message") + check("not a valid show flag: 'foo'", "-level", "-foo", "-message") + check("not a valid show flag: 'foo'", "+level", "+foo", "+message") +} diff --git a/client/go/internal/admin/script-utils/logfmt/tail.go b/client/go/internal/admin/script-utils/logfmt/tail.go new file mode 100644 index 00000000000..75e7cbb0693 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/tail.go @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: mpolden + +package logfmt + +type Line struct { + Text string +} + +type Tail interface { + Lines() chan Line +} diff --git a/client/go/internal/admin/script-utils/logfmt/tail_not_unix.go b/client/go/internal/admin/script-utils/logfmt/tail_not_unix.go new file mode 100644 index 00000000000..7030572575d --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/tail_not_unix.go @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: mpolden + +//go:build windows + +package logfmt + +import ( + "fmt" +) + +func FollowFile(fn string) (Tail, error) { + return nil, fmt.Errorf("tail is not supported on this platform") +} diff --git a/client/go/internal/admin/script-utils/logfmt/tail_unix.go b/client/go/internal/admin/script-utils/logfmt/tail_unix.go new file mode 100644 index 00000000000..7703844da48 --- /dev/null +++ b/client/go/internal/admin/script-utils/logfmt/tail_unix.go @@ -0,0 +1,170 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +//go:build !windows + +package logfmt + +import ( + "bufio" + "fmt" + "io" + "os" + "time" + + "golang.org/x/sys/unix" +) + +const lastLinesSize = 4 * 1024 + +// an active "tail -f" like object + +type unixTail struct { + lines chan Line + lineBuf []byte + curFile *os.File + fn string + reader *bufio.Reader + curStat unix.Stat_t +} + +func (t *unixTail) Lines() chan Line { return t.lines } + +// API for starting to follow a log file + +func FollowFile(fn string) (Tail, error) { + res := unixTail{} + res.fn = fn + res.lineBuf = make([]byte, lastLinesSize) + res.openTail() + res.lines = make(chan Line, 20) + res.lineBuf = res.lineBuf[:0] + go runTailWith(&res) + return &res, nil +} + +func (t *unixTail) setFile(f *os.File) { + if t.curFile != nil { + t.curFile.Close() + } + t.curFile = f + if f != nil { + err := unix.Fstat(int(f.Fd()), &t.curStat) + if err != nil { + f.Close() + fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) + return + } + t.reader = bufio.NewReaderSize(f, 1024*1024) + } else { + t.reader = nil + } +} + +// open log file and seek to the start of a line near the end, if possible. +func (t *unixTail) openTail() { + file, err := os.Open(t.fn) + if err != nil { + return + } + sz, err := file.Seek(0, os.SEEK_END) + if err != nil { + return + } + if sz < lastLinesSize { + sz, err = file.Seek(0, os.SEEK_SET) + if err == nil { + // just read from start of file, all OK + t.setFile(file) + } + return + } + sz, _ = file.Seek(-lastLinesSize, os.SEEK_END) + n, err := file.Read(t.lineBuf) + if err != nil { + return + } + for i := 0; i < n; i++ { + if t.lineBuf[i] == '\n' { + sz, err = file.Seek(sz+int64(i+1), os.SEEK_SET) + if err == nil { + t.setFile(file) + } + return + } + } +} + +func (t *unixTail) reopen(cur *unix.Stat_t) { + for cnt := 0; cnt < 100; cnt++ { + file, err := os.Open(t.fn) + if err != nil { + t.setFile(nil) + if cnt == 0 { + fmt.Fprintf(os.Stderr, "%v (waiting for log file to appear)\n", err) + } + time.Sleep(1000 * time.Millisecond) + continue + } + var stat unix.Stat_t + err = unix.Fstat(int(file.Fd()), &stat) + if err != nil { + file.Close() + fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) + time.Sleep(5000 * time.Millisecond) + continue + } + if cur != nil && cur.Dev == stat.Dev && cur.Ino == stat.Ino { + // same file, continue following it + file.Close() + return + } + // new file, start following it + t.setFile(file) + return + } +} + +// runs as a goroutine +func runTailWith(t *unixTail) { + defer t.setFile(nil) +loop: + for { + for t.curFile == nil { + t.reopen(nil) + } + bytes, err := t.reader.ReadSlice('\n') + t.lineBuf = append(t.lineBuf, bytes...) + if err == bufio.ErrBufferFull { + continue + } + if err == nil { + ll := len(t.lineBuf) - 1 + t.lines <- Line{Text: string(t.lineBuf[:ll])} + t.lineBuf = t.lineBuf[:0] + continue + } + if err == io.EOF { + pos, _ := t.curFile.Seek(0, os.SEEK_CUR) + for cnt := 0; cnt < 100; cnt++ { + time.Sleep(10 * time.Millisecond) + sz, _ := t.curFile.Seek(0, os.SEEK_END) + if sz != pos { + if sz < pos { + // truncation case + pos = 0 + } + t.curFile.Seek(pos, os.SEEK_SET) + continue loop + } + } + // no change in file size, try reopening + t.reopen(&t.curStat) + } else { + fmt.Fprintf(os.Stderr, "error tailing '%s': %v\n", t.fn, err) + close(t.lines) + return + } + } +} diff --git a/client/go/internal/admin/script-utils/main.go b/client/go/internal/admin/script-utils/main.go new file mode 100644 index 00000000000..a6016c86291 --- /dev/null +++ b/client/go/internal/admin/script-utils/main.go @@ -0,0 +1,106 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" + "github.com/vespa-engine/vespa/client/go/internal/admin/script-utils/configserver" + "github.com/vespa-engine/vespa/client/go/internal/admin/script-utils/logfmt" + "github.com/vespa-engine/vespa/client/go/internal/admin/script-utils/services" + "github.com/vespa-engine/vespa/client/go/internal/admin/script-utils/standalone" + "github.com/vespa-engine/vespa/client/go/internal/admin/script-utils/startcbinary" + "github.com/vespa-engine/vespa/client/go/internal/cli/cmd/clusterstate" + "github.com/vespa-engine/vespa/client/go/internal/cli/cmd/deploy" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func basename(s string) string { + parts := strings.Split(s, "/") + return parts[len(parts)-1] +} + +func main() { + defer handleSimplePanic() + _ = vespa.FindAndVerifyVespaHome() + action := basename(os.Args[0]) + if action == "script-utils" && len(os.Args) > 1 { + action = os.Args[1] + os.Args = os.Args[1:] + } + switch action { + case "vespa-stop-services": + os.Exit(services.VespaStopServices()) + case "vespa-start-services": + os.Exit(services.VespaStartServices()) + case "start-services": + os.Exit(services.StartServices()) + case "just-run-configproxy": + os.Exit(services.JustRunConfigproxy()) + case "vespa-start-configserver": + os.Exit(configserver.StartConfigserverEtc()) + case "just-start-configserver": + os.Exit(configserver.JustStartConfigserver()) + case "vespa-start-container-daemon": + os.Exit(jvm.RunApplicationContainer(os.Args[1:])) + case "run-standalone-container": + os.Exit(standalone.StartStandaloneContainer(os.Args[1:])) + case "start-c-binary": + os.Exit(startcbinary.Run(os.Args[1:])) + case "export-env": + vespa.ExportDefaultEnvToSh() + case "security-env", "vespa-security-env": + vespa.ExportSecurityEnvToSh() + case "ipv6-only": + if vespa.HasOnlyIpV6() { + os.Exit(0) + } else { + os.Exit(1) + } + case "detect-hostname": + myName, err := vespa.FindOurHostname() + fmt.Println(myName) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + case "vespa-deploy": + cobra := deploy.NewDeployCmd() + cobra.Execute() + case "vespa-logfmt": + cobra := logfmt.NewLogfmtCmd() + cobra.Execute() + case "vespa-get-cluster-state": + cobra := clusterstate.NewGetClusterStateCmd() + cobra.Execute() + case "vespa-get-node-state": + cobra := clusterstate.NewGetNodeStateCmd() + cobra.Execute() + case "vespa-set-node-state": + cobra := clusterstate.NewSetNodeStateCmd() + cobra.Execute() + default: + if startcbinary.IsCandidate(os.Args[0]) { + os.Exit(startcbinary.Run(os.Args)) + } + fmt.Fprintf(os.Stderr, "unknown action '%s'\n", action) + fmt.Fprintln(os.Stderr, "actions: export-env, ipv6-only, security-env, detect-hostname") + fmt.Fprintln(os.Stderr, "(also: vespa-deploy, vespa-logfmt)") + } +} + +func handleSimplePanic() { + if r := recover(); r != nil { + if jee, ok := r.(*util.JustExitError); ok { + fmt.Fprintln(os.Stderr, jee) + os.Exit(1) + } else { + panic(r) + } + } +} diff --git a/client/go/internal/admin/script-utils/services/configproxy.go b/client/go/internal/admin/script-utils/services/configproxy.go new file mode 100644 index 00000000000..877de387efd --- /dev/null +++ b/client/go/internal/admin/script-utils/services/configproxy.go @@ -0,0 +1,143 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + CONFIGPROXY_PIDFILE = "var/run/configproxy.pid" + PROXY_SERVICE_NAME = "configproxy" +) + +func JustRunConfigproxy() int { + commonPreChecks() + vespa.CheckCorrectUser() + configsources := defaults.VespaConfigserverRpcAddrs() + if len(configsources) < 1 { + util.JustExitMsg("could not find any configservers") + } + util.TuneResourceLimits() + c := jvm.NewConfigProxyJvm(PROXY_SERVICE_NAME) + userargs := os.Getenv(envvars.VESPA_CONFIGPROXY_JVMARGS) + c.ConfigureOptions(configsources, userargs) + c.Exec() + // unreachable: + return 1 +} + +func startProxyWithRunserver() { + commonPreChecks() + vespa.CheckCorrectUser() + configsources := defaults.VespaConfigserverRpcAddrs() + fmt.Printf( + "Starting config proxy using %s as config source(s)\n", + strings.Join(configsources, " and ")) + args := []string{ + "-r", "10", + "-s", PROXY_SERVICE_NAME, + "-p", CONFIGPROXY_PIDFILE, + "--", + "libexec/vespa/script-utils", "just-run-configproxy", + } + cmd := exec.Command("vespa-runserver", args...) + cmd.Stdin = nil + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Start() + p := cmd.Process + if p != nil { + p.Release() + } +} + +func waitForProxyResponse() bool { + hname, _ := vespa.FindOurHostname() + backtick := util.BackTicksWithStderr + start := time.Now() + fmt.Printf("Waiting for config proxy to start\n") + for sleepcount := 0; sleepcount < 1800; sleepcount++ { + time.Sleep(100 * time.Millisecond) + got, err := os.ReadFile(CONFIGPROXY_PIDFILE) + if err == nil { + pid, err := strconv.Atoi(strings.TrimSpace(string(got))) + if err == nil && pid > 0 { + out, err := backtick.Run("vespa-ping-configproxy", "-s", hname) + if err == nil { + secs := time.Since(start).Seconds() + fmt.Printf("config proxy started after %ds (runserver pid %d)\n", int(secs), pid) + return true + } + if sleepcount%50 == 9 { + trace.Warning("Could not ping configproxy:", err) + if sleepcount%500 == 59 { + secs := time.Since(start).Seconds() + trace.Warning("ping output after", int(secs), "seconds:", strings.TrimSpace(out)) + logFile := defaults.VespaLogFile() + cmd := fmt.Sprintf("tail -n 15 %s | vespa-logfmt -l all -N", logFile) + out, err = backtick.Run("sh", "-c", cmd) + fmt.Fprintf(os.Stderr, "tail of logfile: >>>\n%s<<<\n", out) + } + } + } else { + trace.Debug("bad contents (", string(got), ") in pid file", CONFIGPROXY_PIDFILE) + } + } else { + trace.Debug("bad pid file", CONFIGPROXY_PIDFILE, err) + } + } + secs := time.Since(start).Seconds() + fmt.Fprintf(os.Stderr, "Config proxy still failed to start after %d seconds!\n", int(secs)) + got, err := os.ReadFile(CONFIGPROXY_PIDFILE) + if err != nil { + fmt.Fprintf(os.Stderr, "pid file %s was not created\n", CONFIGPROXY_PIDFILE) + return false + } + gotpid := strings.TrimSpace(string(got)) + pid, err := strconv.Atoi(gotpid) + if err != nil || pid < 1 { + fmt.Fprintf(os.Stderr, "invalid pid '%s' in file %s\n", gotpid, CONFIGPROXY_PIDFILE) + return false + } + out, err := backtick.Run("kill", "-0", gotpid) + if err != nil { + fmt.Fprintf(os.Stderr, "config proxy process `%s` has terminated: %s\n", gotpid, strings.TrimSpace(out)) + return false + } + out, err = backtick.Run("vespa-ping-configproxy", "-s", hname) + fmt.Fprintf(os.Stderr, "failed to ping configproxy: %s\n", out) + return false +} + +func StartConfigproxy() int { + commonPreChecks() + vespa.MaybeSwitchUser("start-configproxy") + startProxyWithRunserver() + if waitForProxyResponse() { + return 0 + } + return 1 +} + +func stopProxyWithRunserver() { + _, err := util.SystemCommand.Run("vespa-runserver", + "-s", PROXY_SERVICE_NAME, + "-p", CONFIGPROXY_PIDFILE, "-S") + if err != nil { + trace.Warning("Stopping sentinel:", err) + } +} diff --git a/client/go/internal/admin/script-utils/services/env.go b/client/go/internal/admin/script-utils/services/env.go new file mode 100644 index 00000000000..65dd583b766 --- /dev/null +++ b/client/go/internal/admin/script-utils/services/env.go @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func exportSettings(vespaHome string) { + vlt := fmt.Sprintf("file:%s/logs/vespa/vespa.log", vespaHome) + lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome) + dlp := fmt.Sprintf("%s/lib64:/opt/vespa-deps/lib64", vespaHome) + os.Setenv(envvars.VESPA_LOG_TARGET, vlt) + os.Setenv(envvars.VESPA_LOG_CONTROL_DIR, lcd) + os.Setenv(envvars.LD_LIBRARY_PATH, dlp) + os.Setenv(envvars.JAVAVM_LD_PRELOAD, "") + os.Setenv(envvars.LD_PRELOAD, "") + os.Setenv(envvars.MALLOC_ARENA_MAX, "1") + util.OptionallyReduceTimerFrequency() +} diff --git a/client/go/internal/admin/script-utils/services/prechecks.go b/client/go/internal/admin/script-utils/services/prechecks.go new file mode 100644 index 00000000000..690447a9fee --- /dev/null +++ b/client/go/internal/admin/script-utils/services/prechecks.go @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func commonPreChecks() (veHome, veHost string) { + if doTrace := os.Getenv(envvars.TRACE_STARTUP); doTrace != "" { + trace.AdjustVerbosity(1) + } + if doDebug := os.Getenv(envvars.DEBUG_STARTUP); doDebug != "" { + trace.AdjustVerbosity(2) + } + _ = vespa.FindAndVerifyVespaHome() + err := vespa.LoadDefaultEnv() + if err != nil { + panic(err) + } + veHome = vespa.FindAndVerifyVespaHome() + veHost, err = vespa.FindOurHostname() + if err != nil { + trace.Warning("could not detect hostname:", err, "; using fallback:", veHost) + } + err = os.Chdir(veHome) + if err != nil { + util.JustExitWith(err) + } + return +} diff --git a/client/go/internal/admin/script-utils/services/sentinel.go b/client/go/internal/admin/script-utils/services/sentinel.go new file mode 100644 index 00000000000..1017924263b --- /dev/null +++ b/client/go/internal/admin/script-utils/services/sentinel.go @@ -0,0 +1,93 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + SENTINEL_PIDFILE = "var/run/sentinel.pid" + SENTINEL_SERVICE_NAME = "config-sentinel" +) + +func startSentinelWithRunserver() { + _, veHost := commonPreChecks() + vespa.CheckCorrectUser() + os.Setenv(envvars.VESPA_SERVICE_NAME, SENTINEL_SERVICE_NAME) + configId := fmt.Sprintf("hosts/%s", veHost) + args := []string{ + "-r", "10", + "-s", SENTINEL_SERVICE_NAME, + "-p", SENTINEL_PIDFILE, + "--", + "sbin/vespa-config-sentinel", + "-c", configId, + } + cmd := exec.Command("vespa-runserver", args...) + cmd.Stdin = nil + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Start() + p := cmd.Process + if p != nil { + p.Release() + } + os.Unsetenv(envvars.VESPA_SERVICE_NAME) +} + +func waitForSentinelPid() bool { + backtick := util.BackTicksWithStderr + start := time.Now() + for sleepcount := 0; sleepcount < 1000; sleepcount++ { + time.Sleep(10 * time.Millisecond) + got, err := os.ReadFile(CONFIGPROXY_PIDFILE) + if err == nil { + pid, err := strconv.Atoi(strings.TrimSpace(string(got))) + if err == nil && pid > 0 { + _, err := backtick.Run("kill", "-0", strconv.Itoa(pid)) + if err == nil { + secs := int(time.Since(start).Seconds()) + if secs > 0 { + fmt.Printf("config sentinel started after %d seconds\n", secs) + } + return true + } + } + } + if sleepcount%500 == 90 { + trace.Warning("Waiting for sentinel to start") + } + } + return false +} + +func StartConfigSentinel() int { + commonPreChecks() + vespa.MaybeSwitchUser("start-config-sentinel") + startSentinelWithRunserver() + if waitForSentinelPid() { + return 0 + } + return 1 +} + +func stopSentinelWithRunserver() { + _, err := util.SystemCommand.Run("vespa-runserver", + "-s", SENTINEL_SERVICE_NAME, + "-p", SENTINEL_PIDFILE, "-S") + if err != nil { + trace.Warning("Stopping sentinel:", err) + } +} diff --git a/client/go/internal/admin/script-utils/services/start.go b/client/go/internal/admin/script-utils/services/start.go new file mode 100644 index 00000000000..f47d99714f1 --- /dev/null +++ b/client/go/internal/admin/script-utils/services/start.go @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "fmt" + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func StartServices() int { + veHome, _ := commonPreChecks() + vespa.CheckCorrectUser() + trace.Debug("running as correct user") + exportSettings(veHome) + if vespa.HasOnlyIpV6() { + os.Setenv(envvars.VESPA_ONLY_IP_V6_NETWORKING, "true") + } + startProxyWithRunserver() + if waitForProxyResponse() { + startSentinelWithRunserver() + if waitForSentinelPid() { + return 0 + } + } + return 1 +} + +func checkjava() { + backticks := util.BackTicksWithStderr + out, err := backticks.Run("java", "-version") + if err != nil { + trace.Warning("cannot run 'java -version'") + util.JustExitWith(err) + } + if !strings.Contains(out, "64-Bit Server VM") { + util.JustExitWith(fmt.Errorf("java must invoke the 64-bit Java VM, but -version says:\n%s\n", out)) + } +} + +func runvalidation() { + // not implemented +} + +func VespaStartServices() int { + home, host := commonPreChecks() + trace.Debug("common prechecks ok, running in", home, "on", host) + vespa.RunPreStart() + trace.Debug("prestart ok") + util.TuneResourceLimits() + trace.Debug("resource limits ok") + checkjava() + trace.Debug("java ok") + runvalidation() + enable_transparent_hugepages_with_background_compaction() + disable_vm_zone_reclaim_mode() + drop_caches() + err := vespa.MaybeSwitchUser("start-services") + if err != nil { + util.JustExitWith(err) + } + return StartServices() +} diff --git a/client/go/internal/admin/script-utils/services/stop.go b/client/go/internal/admin/script-utils/services/stop.go new file mode 100644 index 00000000000..f5b764d122e --- /dev/null +++ b/client/go/internal/admin/script-utils/services/stop.go @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func VespaStopServices() int { + if doTrace := os.Getenv(envvars.TRACE_STARTUP); doTrace != "" { + trace.AdjustVerbosity(1) + } + if doDebug := os.Getenv(envvars.DEBUG_STARTUP); doDebug != "" { + trace.AdjustVerbosity(2) + } + err := vespa.LoadDefaultEnv() + if err != nil { + util.JustExitWith(err) + } + err = vespa.MaybeSwitchUser("vespa-stop-services") + if err != nil { + util.JustExitWith(err) + } + vespa.CheckCorrectUser() + trace.Debug("running as correct user") + stopSentinelWithRunserver() + stopProxyWithRunserver() + return 0 +} diff --git a/client/go/internal/admin/script-utils/services/tuning.go b/client/go/internal/admin/script-utils/services/tuning.go new file mode 100644 index 00000000000..f922495812f --- /dev/null +++ b/client/go/internal/admin/script-utils/services/tuning.go @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +const ( + DROP_CACHES_CTL = "/proc/sys/vm/drop_caches" + TRANSPARENT_HUGEPAGE_ENABLED = "/sys/kernel/mm/transparent_hugepage/enabled" + TRANSPARENT_HUGEPAGE_DEFRAG = "/sys/kernel/mm/transparent_hugepage/defrag" + TRANSPARENT_HUGEPAGE_KH_DEFRAG = "/sys/kernel/mm/transparent_hugepage/khugepaged/defrag" + ZONE_RECLAIM_CTL = "/proc/sys/vm/zone_reclaim_mode" +) + +func maybeEcho(fileName, toWrite string) bool { + f, err := os.OpenFile(fileName, os.O_WRONLY, 0) + if err == nil { + _, err = fmt.Fprintf(f, "%s\n", toWrite) + f.Close() + if err == nil { + trace.Debug("wrote", toWrite, "to", fileName) + return true + } else { + trace.Warning(err) + } + } + return false +} + +func enable_transparent_hugepages_with_background_compaction() { + // Should probably also be done on host. + maybeEcho(TRANSPARENT_HUGEPAGE_ENABLED, "always") + maybeEcho(TRANSPARENT_HUGEPAGE_DEFRAG, "never") + maybeEcho(TRANSPARENT_HUGEPAGE_KH_DEFRAG, "1") +} + +func disable_vm_zone_reclaim_mode() { + maybeEcho(ZONE_RECLAIM_CTL, "0") +} + +func drop_caches() { + if maybeEcho(DROP_CACHES_CTL, "3") { + trace.Debug("dropped caches") + } +} diff --git a/client/go/internal/admin/script-utils/standalone/start.go b/client/go/internal/admin/script-utils/standalone/start.go new file mode 100644 index 00000000000..a26ac8e9d8c --- /dev/null +++ b/client/go/internal/admin/script-utils/standalone/start.go @@ -0,0 +1,53 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +// for starting standalone jdisc containers +package standalone + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func commonPreChecks() { + if doTrace := os.Getenv("TRACE_JVM_STARTUP"); doTrace != "" { + trace.AdjustVerbosity(1) + } + if doDebug := os.Getenv("DEBUG_JVM_STARTUP"); doDebug != "" { + trace.AdjustVerbosity(2) + } + veHome := vespa.FindAndVerifyVespaHome() + err := os.Chdir(veHome) + if err != nil { + util.JustExitWith(err) + } + err = vespa.LoadDefaultEnv() + if err != nil { + util.JustExitWith(err) + } +} + +func StartStandaloneContainer(extraArgs []string) int { + commonPreChecks() + util.TuneResourceLimits() + serviceName := os.Getenv("VESPA_SERVICE_NAME") + if serviceName == "" { + util.JustExitMsg("Missing service name, ensure VESPA_SERVICE_NAME is set in the environment") + } + c := jvm.NewStandaloneContainer(serviceName) + jvmOpts := c.JvmOptions() + jvmOpts.AddOption("-DOnnxBundleActivator.skip=true") + for _, extra := range extraArgs { + jvmOpts.AddOption(extra) + } + minFallback := jvm.MegaBytesOfMemory(128) + maxFallback := jvm.MegaBytesOfMemory(2048) + jvmOpts.AddDefaultHeapSizeArgs(minFallback, maxFallback) + c.Exec() + // unreachable: + return 1 +} diff --git a/client/go/internal/admin/script-utils/startcbinary/cmd.go b/client/go/internal/admin/script-utils/startcbinary/cmd.go new file mode 100644 index 00000000000..9580a9240bb --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/cmd.go @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func Run(args []string) int { + trace.AdjustVerbosity(0) + if len(args) < 1 { + trace.Warning("missing program argument") + return 1 + } + spec := NewProgSpec(args) + err := vespa.LoadDefaultEnv() + if err != nil { + util.JustExitWith(err) + } + hostname, err := vespa.FindOurHostname() + if err != nil { + trace.Warning("could not detect hostname:", err, "; using fallback:", hostname) + } + return startCbinary(spec) +} + +func IsCandidate(program string) bool { + binary := program + "-bin" + if strings.Contains(binary, "/") { + return util.IsRegularFile(binary) + } else { + path := strings.Split(os.Getenv(envvars.PATH), ":") + for _, dir := range path { + fn := dir + "/" + binary + if util.IsRegularFile(fn) { + return true + } + } + } + return false +} diff --git a/client/go/internal/admin/script-utils/startcbinary/common_env.go b/client/go/internal/admin/script-utils/startcbinary/common_env.go new file mode 100644 index 00000000000..6bc730b5119 --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/common_env.go @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func (spec *ProgSpec) considerFallback(varName, varValue string) { + if spec.getenv(varName) == "" && varValue != "" { + spec.setenv(varName, varValue) + } +} + +func (spec *ProgSpec) considerEnvFallback(targetVar, fallbackVar string) { + spec.considerFallback(targetVar, spec.getenv(fallbackVar)) +} + +func (spec *ProgSpec) configureCommonEnv() { + os.Unsetenv(envvars.LD_PRELOAD) + spec.setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") + spec.setenv(envvars.GLIBCXX_FORCE_NEW, "1") + spec.setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") + spec.setenv(envvars.MALLOC_ARENA_MAX, "1") + + // fallback from old env.vars: + spec.considerEnvFallback(envvars.VESPA_USE_HUGEPAGES_LIST, envvars.HUGEPAGES_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_MADVISE_LIST, envvars.MADVISE_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC, envvars.VESPAMALLOC_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_D, envvars.VESPAMALLOCD_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_DST, envvars.VESPAMALLOCDST_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_NO_VESPAMALLOC, envvars.NO_VESPAMALLOC_LIST) + // other fallbacks: + spec.considerFallback(envvars.ROOT, vespa.FindHome()) + spec.considerFallback(envvars.VESPA_USER, vespa.FindVespaUser()) + spec.considerFallback(envvars.VESPA_USE_HUGEPAGES_LIST, "all") + spec.considerFallback(envvars.VESPA_USE_VESPAMALLOC, "all") + spec.considerFallback(envvars.VESPA_USE_NO_VESPAMALLOC, strings.Join([]string{ + "vespa-rpc-invoke", + "vespa-get-config", + "vespa-sentinel-cmd", + "vespa-route", + "vespa-proton-cmd", + "vespa-configproxy-cmd", + "vespa-config-status", + }, " ")) + +} + +func (spec *ProgSpec) configureHugePages() { + if spec.matchesListEnv(envvars.VESPA_USE_HUGEPAGES_LIST) { + spec.setenv(envvars.VESPA_USE_HUGEPAGES, "yes") + } +} + +func (spec *ProgSpec) configureUseMadvise() { + limit := spec.valueFromListEnv(envvars.VESPA_USE_MADVISE_LIST) + if limit != "" { + trace.Trace("shall use madvise with limit", limit, "as set in", envvars.VESPA_USE_MADVISE_LIST) + spec.setenv(envvars.VESPA_MALLOC_MADVISE_LIMIT, limit) + return + } +} + +func (spec *ProgSpec) configurePath() { + // Prefer newer gdb and pstack: + spec.prependPath("/opt/rh/gcc-toolset-12/root/usr/bin") + // Maven is needed for tester applications: + spec.prependPath(vespa.FindHome() + "/local/maven/bin") + spec.prependPath(vespa.FindHome() + "/bin64") + spec.prependPath(vespa.FindHome() + "/bin") + // how to find the "java" program? + // should be available in $VESPA_HOME/bin or JAVA_HOME + if javaHome := spec.getenv(envvars.JAVA_HOME); javaHome != "" { + spec.prependPath(javaHome + "/bin") + } +} diff --git a/client/go/internal/admin/script-utils/startcbinary/mockbin/bad-numactl b/client/go/internal/admin/script-utils/startcbinary/mockbin/bad-numactl new file mode 100755 index 00000000000..1b08bb19b95 --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/mockbin/bad-numactl @@ -0,0 +1,25 @@ +#!/bin/sh + +# emulate numactl which says hardware is available but fails anyway + +case $#.$* in + 1.--hardware) + cat << 'EOF' +available: 2 nodes (0-1) +node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 +node 0 size: 128604 MB +node 0 free: 107167 MB +node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39 41 43 45 47 +node 1 size: 128976 MB +node 1 free: 110921 MB +node distances: +node 0 1 + 0: 10 21 + 1: 21 10 +EOF + ;; + *) + echo "numactl failure" + ;; +esac +exit 0 diff --git a/client/go/internal/admin/script-utils/startcbinary/mockbin/good-numactl b/client/go/internal/admin/script-utils/startcbinary/mockbin/good-numactl new file mode 100755 index 00000000000..f861809b382 --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/mockbin/good-numactl @@ -0,0 +1,25 @@ +#!/bin/sh + +# emulate normal numactl operations + +case $#.$* in + 1.--hardware) + cat << 'EOF' +available: 2 nodes (0-1) +node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 +node 0 size: 128604 MB +node 0 free: 107167 MB +node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39 41 43 45 47 +node 1 size: 128976 MB +node 1 free: 110921 MB +node distances: +node 0 1 + 0: 10 21 + 1: 21 10 +EOF + ;; + *) + echo "foo" + ;; +esac +exit 0 diff --git a/client/go/internal/admin/script-utils/startcbinary/mockbin/has-valgrind b/client/go/internal/admin/script-utils/startcbinary/mockbin/has-valgrind new file mode 100755 index 00000000000..8716209928e --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/mockbin/has-valgrind @@ -0,0 +1,6 @@ +#!/bin/sh + +# emulate "which" that finds valgrind + +echo "/opt/vespa-deps/bin/valgrind" +exit 0 diff --git a/client/go/internal/admin/script-utils/startcbinary/mockbin/no-numactl b/client/go/internal/admin/script-utils/startcbinary/mockbin/no-numactl new file mode 100755 index 00000000000..dfe583abaca --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/mockbin/no-numactl @@ -0,0 +1,6 @@ +#!/bin/sh + +# emulate inoperational numactl + +echo "No NUMA available on this system" +exit 1 diff --git a/client/go/internal/admin/script-utils/startcbinary/mockbin/no-valgrind b/client/go/internal/admin/script-utils/startcbinary/mockbin/no-valgrind new file mode 100755 index 00000000000..7a74e29b84b --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/mockbin/no-valgrind @@ -0,0 +1,6 @@ +#!/bin/sh + +# emulate "which" that does not find valgrind + +echo "which: no valgrind in ($PATH)" >&2 +exit 1 diff --git a/client/go/internal/admin/script-utils/startcbinary/numactl.go b/client/go/internal/admin/script-utils/startcbinary/numactl.go new file mode 100644 index 00000000000..fe091dedba9 --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/numactl.go @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "fmt" + "strconv" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func (p *ProgSpec) configureNumaCtl() { + p.shouldUseNumaCtl = false + p.numaSocket = -1 + if p.getenv(envvars.VESPA_NO_NUMACTL) != "" { + return + } + backticks := util.BackTicksIgnoreStderr + out, err := backticks.Run("numactl", "--hardware") + trace.Debug("numactl --hardware says:", out) + if err != nil { + trace.Trace("numactl error:", err) + return + } + outfoo, errfoo := backticks.Run("numactl", "--interleave", "all", "echo", "foo") + if errfoo != nil { + trace.Trace("cannot run with numactl:", errfoo) + return + } + if outfoo != "foo\n" { + trace.Trace("bad numactl output:", outfoo) + return + } + p.shouldUseNumaCtl = true + if affinity := p.getenv(envvars.VESPA_AFFINITY_CPU_SOCKET); affinity != "" { + wantSocket, _ := strconv.Atoi(affinity) + trace.Debug("want socket:", wantSocket) + parts := strings.Fields(out) + for idx := 0; idx+2 < len(parts); idx++ { + if parts[idx] == "available:" && parts[idx+2] == "nodes" { + numSockets, _ := strconv.Atoi(parts[idx+1]) + trace.Debug("numSockets:", numSockets) + if numSockets > 1 { + p.numaSocket = (wantSocket % numSockets) + return + } + } + } + } +} + +func (p *ProgSpec) numaCtlBinary() string { + return "numactl" +} + +func (p *ProgSpec) prependNumaCtl(args []string) []string { + v := util.NewArrayList[string](5 + len(args)) + v.Append("numactl") + if p.numaSocket >= 0 { + v.Append(fmt.Sprintf("--cpunodebind=%d", p.numaSocket)) + v.Append(fmt.Sprintf("--membind=%d", p.numaSocket)) + } else { + v.Append("--interleave") + v.Append("all") + } + v.AppendAll(args...) + return v +} diff --git a/client/go/internal/admin/script-utils/startcbinary/numactl_test.go b/client/go/internal/admin/script-utils/startcbinary/numactl_test.go new file mode 100644 index 00000000000..65f52be988e --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/numactl_test.go @@ -0,0 +1,89 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package startcbinary + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +func setup(t *testing.T, testFileName string) { + trace.AdjustVerbosity(1) + mockBinParent = strings.TrimSuffix(testFileName, "/numactl_test.go") + tmpBin = t.TempDir() + "/mock.bin.numactl_test" + err := os.MkdirAll(tmpBin, 0755) + assert.Nil(t, err) + t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH"))) +} + +func TestNumaCtlDetection(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + _, tfn, _, _ := runtime.Caller(0) + setup(t, tfn) + orig := []string{"/bin/myprog", "-c", "cfgid"} + spec := NewProgSpec(orig) + + useMock("no-numactl", "numactl") + spec.configureNumaCtl() + assert.Equal(t, false, spec.shouldUseNumaCtl) + + useMock("bad-numactl", "numactl") + spec.configureNumaCtl() + assert.Equal(t, false, spec.shouldUseNumaCtl) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "") + useMock("good-numactl", "numactl") + spec.configureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, -1, spec.numaSocket) + argv := spec.prependNumaCtl(orig) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "numactl", argv[0]) + assert.Equal(t, "--interleave", argv[1]) + assert.Equal(t, "all", argv[2]) + assert.Equal(t, "/bin/myprog-bin", argv[3]) + assert.Equal(t, "-c", argv[4]) + assert.Equal(t, "cfgid", argv[5]) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "0") + spec.configureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, 0, spec.numaSocket) + argv = spec.prependNumaCtl(orig) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "numactl", argv[0]) + assert.Equal(t, "--cpunodebind=0", argv[1]) + assert.Equal(t, "--membind=0", argv[2]) + assert.Equal(t, "/bin/myprog-bin", argv[3]) + assert.Equal(t, "-c", argv[4]) + assert.Equal(t, "cfgid", argv[5]) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "1") + spec.configureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, 1, spec.numaSocket) + argv = spec.prependNumaCtl(orig) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "numactl", argv[0]) + assert.Equal(t, "--cpunodebind=1", argv[1]) + assert.Equal(t, "--membind=1", argv[2]) + assert.Equal(t, "/bin/myprog-bin", argv[3]) + assert.Equal(t, "-c", argv[4]) + assert.Equal(t, "cfgid", argv[5]) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "2") + spec.configureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, 0, spec.numaSocket) + +} diff --git a/client/go/internal/admin/script-utils/startcbinary/progspec.go b/client/go/internal/admin/script-utils/startcbinary/progspec.go new file mode 100644 index 00000000000..9975f6c3c90 --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/progspec.go @@ -0,0 +1,144 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +type ProgSpec struct { + Program string + Args []string + BaseName string + Env map[string]string + numaSocket int + shouldUseCallgrind bool + shouldUseValgrind bool + shouldUseNumaCtl bool + shouldUseVespaMalloc bool + vespaMallocPreload string +} + +func NewProgSpec(argv []string) *ProgSpec { + progName := argv[0] + binProg := progName + "-bin" + p := ProgSpec{ + Program: binProg, + Args: argv, + BaseName: baseNameOf(progName), + Env: make(map[string]string), + numaSocket: -1, + } + p.Args[0] = binProg + return &p +} + +func baseNameOf(s string) string { + idx := strings.LastIndex(s, "/") + idx++ + return s[idx:] +} + +func (p *ProgSpec) setenv(k, v string) { + p.Env[k] = v +} + +func (p *ProgSpec) getenv(k string) string { + if v, ok := p.Env[k]; ok { + return v + } + return os.Getenv(k) +} + +func (p *ProgSpec) prependPath(dirName string) { + pathList := []string{dirName} + oldPath := p.getenv(envvars.PATH) + if oldPath == "" { + oldPath = "/usr/bin" + } + for _, part := range strings.Split(oldPath, ":") { + if part != dirName { + pathList = append(pathList, part) + } + } + newPath := strings.Join(pathList, ":") + p.setenv(envvars.PATH, newPath) + os.Setenv(envvars.PATH, newPath) +} + +func (p *ProgSpec) matchesListEnv(envVarName string) bool { + return p.matchesListString(p.getenv(envVarName)) +} + +func (p *ProgSpec) matchesListString(env string) bool { + if env == "all" { + trace.Debug(p.Program, "always matching in:", env) + return true + } + parts := strings.Fields(env) + for _, part := range parts { + if p.BaseName == part { + trace.Debug(p.Program, "has basename matching in:", env) + return true + } + trace.Debug("checking matching:", p.BaseName, "!=", part) + } + return false +} + +func (p *ProgSpec) valueFromListEnv(envVarName string) string { + return p.valueFromListString(p.getenv(envVarName)) +} + +func (p *ProgSpec) valueFromListString(env string) string { + parts := strings.Fields(env) + for _, part := range parts { + idx := strings.Index(part, "=") + if idx <= 0 { + trace.Trace("expected key=value, but got:", part) + continue + } + partName := part[:idx] + idx++ + partValue := part[idx:] + if p.BaseName == partName || partName == "all" { + trace.Debug(p.Program, "has basename matching in:", env) + return partValue + } + trace.Debug("checking matching:", p.BaseName, "!=", part) + } + return "" +} + +func (spec *ProgSpec) effectiveEnv() []string { + env := make(map[string]string) + for _, entry := range os.Environ() { + addInMap := func(kv string) bool { + for idx, elem := range kv { + if elem == '=' { + k := kv[:idx] + env[k] = kv + return true + } + } + return false + } + if !addInMap(entry) { + env[entry] = "" + } + } + for k, v := range spec.Env { + trace.Trace("add to environment:", k, "=", v) + env[k] = k + "=" + v + } + envv := make([]string, 0, len(env)) + for _, v := range env { + envv = append(envv, v) + } + return envv +} diff --git a/client/go/internal/admin/script-utils/startcbinary/progspec_test.go b/client/go/internal/admin/script-utils/startcbinary/progspec_test.go new file mode 100644 index 00000000000..be113e4e350 --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/progspec_test.go @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package startcbinary + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProgSpec(t *testing.T) { + spec := NewProgSpec([]string{"/opt/vespa/bin/foobar"}) + var b bool + + b = spec.matchesListString("") + assert.Equal(t, false, b) + b = spec.matchesListString("foobar") + assert.Equal(t, true, b) + b = spec.matchesListString("foo bar") + assert.Equal(t, false, b) + b = spec.matchesListString("one foobar") + assert.Equal(t, true, b) + b = spec.matchesListString("foobar two") + assert.Equal(t, true, b) + b = spec.matchesListString("one foobar two") + assert.Equal(t, true, b) + b = spec.matchesListString("all") + assert.Equal(t, true, b) + + var s string + s = spec.valueFromListString("") + assert.Equal(t, "", s) + s = spec.valueFromListString("foobar=123") + assert.Equal(t, "123", s) + s = spec.valueFromListString("one=1 foobar=123 two=2") + assert.Equal(t, "123", s) + s = spec.valueFromListString("one=1 all=123") + assert.Equal(t, "123", s) +} diff --git a/client/go/internal/admin/script-utils/startcbinary/startcbinary.go b/client/go/internal/admin/script-utils/startcbinary/startcbinary.go new file mode 100644 index 00000000000..f5e58e59808 --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/startcbinary.go @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func startCbinary(spec *ProgSpec) int { + spec.configureCommonEnv() + spec.configurePath() + spec.configureTuning() + spec.configureValgrind() + spec.configureNumaCtl() + spec.configureHugePages() + spec.configureUseMadvise() + spec.configureVespaMalloc() + err := spec.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } else { + return 0 + } +} + +func (spec *ProgSpec) run() error { + prog := spec.Program + args := spec.Args + if spec.shouldUseValgrind { + args = spec.prependValgrind(args) + prog = spec.valgrindBinary() + } else if spec.shouldUseNumaCtl { + args = spec.prependNumaCtl(args) + prog = spec.numaCtlBinary() + } + if spec.shouldUseVespaMalloc { + spec.setenv(envvars.LD_PRELOAD, spec.vespaMallocPreload) + } + envv := spec.effectiveEnv() + return util.Execvpe(prog, args, envv) +} diff --git a/client/go/internal/admin/script-utils/startcbinary/tuning.go b/client/go/internal/admin/script-utils/startcbinary/tuning.go new file mode 100644 index 00000000000..57230629d7a --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/tuning.go @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func (spec *ProgSpec) configureTuning() { + util.OptionallyReduceTimerFrequency() + util.TuneResourceLimits() +} diff --git a/client/go/internal/admin/script-utils/startcbinary/valgrind.go b/client/go/internal/admin/script-utils/startcbinary/valgrind.go new file mode 100644 index 00000000000..43a1ed602bd --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/valgrind.go @@ -0,0 +1,82 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "fmt" + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func (p *ProgSpec) configureValgrind() { + p.shouldUseValgrind = false + p.shouldUseCallgrind = false + env := p.getenv(envvars.VESPA_USE_VALGRIND) + parts := strings.Split(env, " ") + for _, part := range parts { + if p.BaseName == part { + trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env) + backticks := util.BackTicksWithStderr + out, err := backticks.Run("which", "valgrind") + if err != nil { + trace.Trace("no valgrind, 'which' fails:", err, "=>", out) + return + } + if opts := p.getenv(envvars.VESPA_VALGRIND_OPT); strings.Contains(opts, "callgrind") { + p.shouldUseCallgrind = true + } + p.shouldUseValgrind = true + return + } + trace.Debug("checking", envvars.VESPA_USE_VALGRIND, ":", p.BaseName, "!=", part) + } +} + +func (p *ProgSpec) valgrindBinary() string { + return "valgrind" +} + +func (p *ProgSpec) valgrindOptions() []string { + env := p.getenv(envvars.VESPA_VALGRIND_OPT) + if env != "" { + return strings.Fields(env) + } + result := []string{ + "--num-callers=32", + "--run-libc-freeres=yes", + "--track-origins=yes", + "--freelist-vol=1000000000", + "--leak-check=full", + "--show-reachable=yes", + } + result = addValgrindSuppression(result, "etc/vespa/valgrind-suppressions.txt") + result = addValgrindSuppression(result, "etc/vespa/suppressions.txt") + return result +} + +func addValgrindSuppression(r []string, fn string) []string { + existsOk, fileName := vespa.HasFileUnderVespaHome(fn) + if existsOk { + r = append(r, fmt.Sprintf("--suppressions=%s", fileName)) + } + return r +} + +func (p *ProgSpec) valgrindLogOption() string { + return fmt.Sprintf("--log-file=%s/tmp/valgrind.%s.log.%d", vespa.FindHome(), p.BaseName, os.Getpid()) +} + +func (p *ProgSpec) prependValgrind(args []string) []string { + v := util.NewArrayList[string](15 + len(args)) + v.Append(p.valgrindBinary()) + v.AppendAll(p.valgrindOptions()...) + v.Append(p.valgrindLogOption()) + v.AppendAll(args...) + return v +} diff --git a/client/go/internal/admin/script-utils/startcbinary/valgrind_test.go b/client/go/internal/admin/script-utils/startcbinary/valgrind_test.go new file mode 100644 index 00000000000..48cc78474ed --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/valgrind_test.go @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package startcbinary + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +var tmpBin string +var mockBinParent string + +func useMock(prog, target string) { + mock := fmt.Sprintf("%s/mockbin/%s", mockBinParent, prog) + symlink := fmt.Sprintf("%s/%s", tmpBin, target) + os.Remove(symlink) + err := os.Symlink(mock, symlink) + if err != nil { + util.JustExitWith(err) + } +} + +func setupValgrind(t *testing.T, testFileName string) { + trace.AdjustVerbosity(1) + t.Setenv("VESPA_HOME", mockBinParent+"/mock_vespahome") + mockBinParent = strings.TrimSuffix(testFileName, "/valgrind_test.go") + tmpBin = t.TempDir() + "/mock.bin.valgrind_test" + err := os.MkdirAll(tmpBin, 0755) + assert.Nil(t, err) + t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH"))) +} + +func TestValgrindDetection(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + _, tfn, _, _ := runtime.Caller(0) + setupValgrind(t, tfn) + spec := NewProgSpec([]string{"/opt/vespa/bin/foobar"}) + var argv []string + + useMock("has-valgrind", "which") + + t.Setenv("VESPA_USE_VALGRIND", "") + spec.configureValgrind() + assert.Equal(t, false, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + t.Setenv("VESPA_USE_VALGRIND", "foo bar") + spec.configureValgrind() + assert.Equal(t, false, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + t.Setenv("VESPA_USE_VALGRIND", "foobar") + spec.configureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"}) + trace.Trace("argv:", argv) + assert.Equal(t, 11, len(argv)) + assert.Equal(t, "valgrind", argv[0]) + assert.Equal(t, "/bin/myprog", argv[8]) + + t.Setenv("VESPA_USE_VALGRIND", "another foobar yetmore") + spec.configureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + t.Setenv("VESPA_VALGRIND_OPT", "--tool=callgrind") + spec.configureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, true, spec.shouldUseCallgrind) + + argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"}) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "valgrind", argv[0]) + assert.Equal(t, "--tool=callgrind", argv[1]) + assert.Equal(t, "/bin/myprog", argv[3]) + + useMock("no-valgrind", "which") + spec.configureValgrind() + assert.Equal(t, false, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) +} diff --git a/client/go/internal/admin/script-utils/startcbinary/vespamalloc.go b/client/go/internal/admin/script-utils/startcbinary/vespamalloc.go new file mode 100644 index 00000000000..c6d53e1d03c --- /dev/null +++ b/client/go/internal/admin/script-utils/startcbinary/vespamalloc.go @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func vespaMallocLib(suf string) string { + prefixes := []string{"lib64", "lib"} + for _, pre := range prefixes { + fn := fmt.Sprintf("%s/vespa/malloc/%s", pre, suf) + existsOk, fileName := vespa.HasFileUnderVespaHome(fn) + if existsOk { + trace.Debug("found library:", fileName) + return fileName + } + trace.Debug("bad or missing library:", fn) + } + return "" +} + +func (p *ProgSpec) configureVespaMalloc() { + p.shouldUseVespaMalloc = false + if p.matchesListEnv(envvars.VESPA_USE_NO_VESPAMALLOC) { + trace.Trace("use no vespamalloc:", p.BaseName) + return + } + if p.shouldUseValgrind && !p.shouldUseCallgrind { + trace.Trace("use valgrind, so no vespamalloc:", p.BaseName) + return + } + var useFile string + if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_DST) { + useFile = vespaMallocLib("libvespamallocdst16.so") + } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_D) { + useFile = vespaMallocLib("libvespamallocd.so") + } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC) { + useFile = vespaMallocLib("libvespamalloc.so") + } + trace.Trace("use file:", useFile) + if useFile == "" { + return + } + if loadAsHuge := p.getenv(envvars.VESPA_LOAD_CODE_AS_HUGEPAGES); loadAsHuge != "" { + otherFile := vespaMallocLib("libvespa_load_as_huge.so") + useFile = fmt.Sprintf("%s:%s", useFile, otherFile) + } + p.considerEnvFallback(envvars.VESPA_MALLOC_HUGEPAGES, envvars.VESPA_USE_HUGEPAGES) + p.vespaMallocPreload = useFile + p.shouldUseVespaMalloc = true +} diff --git a/client/go/internal/admin/trace/log.go b/client/go/internal/admin/trace/log.go new file mode 100644 index 00000000000..0254d0e4f4a --- /dev/null +++ b/client/go/internal/admin/trace/log.go @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +// handling of informational output +package trace + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" +) + +// make a vespa-format log line + +func logMessage(l outputLevel, msg string) { + out := os.Stderr + unixTime := float64(time.Now().UnixMicro()) * 1.0e-6 + hostname := os.Getenv(envvars.VESPA_HOSTNAME) + pid := os.Getpid() + service := os.Getenv(envvars.VESPA_SERVICE_NAME) + if service == "" { + service = "-" + } + component := "stderr" + level := "error" + switch l { + case levelWarning: + level = "warning" + case levelInfo: + level = "info" + case levelTrace: + level = "info" + msg = fmt.Sprintf("[trace] %s", msg) + case levelDebug: + level = "debug" + case levelSpam: + level = "spam" + } + if !strings.HasSuffix(msg, "\n") { + msg = msg + "\n" + } + fmt.Fprintf(out, "%.6f\t%s\t%d\t%s\t%s\t%s\t%s", + unixTime, hostname, pid, service, component, level, msg) +} + +func LogInfo(msg string) { + logMessage(levelInfo, msg) +} + +func LogDebug(msg string) { + logMessage(levelDebug, msg) +} + +func LogWarning(msg string) { + logMessage(levelWarning, msg) +} diff --git a/client/go/internal/admin/trace/trace.go b/client/go/internal/admin/trace/trace.go new file mode 100644 index 00000000000..bf180e80dad --- /dev/null +++ b/client/go/internal/admin/trace/trace.go @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +// handling of informational output +package trace + +import ( + "fmt" +) + +type outputLevel int + +const ( + levelError outputLevel = iota - 2 + levelWarning + levelNone + levelInfo + levelTrace + levelDebug + levelSpam +) + +var currentOutputLevel outputLevel = levelInfo + +func AdjustVerbosity(howMuch int) { + currentOutputLevel = (outputLevel)(howMuch + int(currentOutputLevel)) +} + +func Silent() { + currentOutputLevel = levelNone +} + +func outputTracing(l outputLevel, v ...interface{}) { + if l > currentOutputLevel { + return + } + msg := fmt.Sprintln(v...) + logMessage(l, msg) +} + +func Info(v ...interface{}) { + outputTracing(levelInfo, v...) +} + +func Trace(v ...interface{}) { + outputTracing(levelTrace, v...) +} + +func Debug(v ...interface{}) { + outputTracing(levelDebug, v...) +} + +func SpamDebug(v ...interface{}) { + outputTracing(levelSpam, v...) +} + +func Warning(v ...interface{}) { + outputTracing(levelWarning, v...) +} |