From 519daa77057387c60a015340b871a62c65fb04ac Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 3 Feb 2023 15:45:56 +0100 Subject: Rename script-utils -> vespa-wrapper --- client/CMakeLists.txt | 60 +- .../admin/script-utils/configserver/check.go | 24 - .../admin/script-utils/configserver/env.go | 31 - .../configserver/fix_dirs_and_files.go | 27 - .../admin/script-utils/configserver/logd.go | 25 - .../admin/script-utils/configserver/runserver.go | 59 -- .../admin/script-utils/configserver/start.go | 86 --- .../internal/admin/script-utils/configserver/zk.go | 22 - .../go/internal/admin/script-utils/logfmt/cmd.go | 46 -- .../admin/script-utils/logfmt/formatflags.go | 41 -- .../admin/script-utils/logfmt/formatflags_test.go | 30 - .../admin/script-utils/logfmt/handleline.go | 179 ----- .../internal/admin/script-utils/logfmt/internal.go | 106 --- .../admin/script-utils/logfmt/internal_names.txt | 782 --------------------- .../script-utils/logfmt/internal_notnames.txt | 8 - .../admin/script-utils/logfmt/internal_test.go | 34 - .../admin/script-utils/logfmt/levelflags.go | 72 -- .../admin/script-utils/logfmt/levelflags_test.go | 74 -- .../internal/admin/script-utils/logfmt/options.go | 48 -- .../admin/script-utils/logfmt/plusminusflag.go | 67 -- .../admin/script-utils/logfmt/regexflag.go | 38 - .../admin/script-utils/logfmt/regexflag_test.go | 41 -- .../admin/script-utils/logfmt/runlogfmt.go | 85 --- .../admin/script-utils/logfmt/showflags.go | 76 -- .../admin/script-utils/logfmt/showflags_test.go | 61 -- .../go/internal/admin/script-utils/logfmt/tail.go | 13 - .../admin/script-utils/logfmt/tail_not_unix.go | 15 - .../admin/script-utils/logfmt/tail_unix.go | 170 ----- client/go/internal/admin/script-utils/main.go | 106 --- .../admin/script-utils/services/configproxy.go | 143 ---- .../go/internal/admin/script-utils/services/env.go | 25 - .../admin/script-utils/services/prechecks.go | 37 - .../admin/script-utils/services/sentinel.go | 93 --- .../internal/admin/script-utils/services/start.go | 69 -- .../internal/admin/script-utils/services/stop.go | 35 - .../internal/admin/script-utils/services/tuning.go | 51 -- .../admin/script-utils/standalone/start.go | 53 -- .../admin/script-utils/startcbinary/cmd.go | 48 -- .../admin/script-utils/startcbinary/common_env.go | 83 --- .../script-utils/startcbinary/mockbin/bad-numactl | 25 - .../script-utils/startcbinary/mockbin/good-numactl | 25 - .../script-utils/startcbinary/mockbin/has-valgrind | 6 - .../script-utils/startcbinary/mockbin/no-numactl | 6 - .../script-utils/startcbinary/mockbin/no-valgrind | 6 - .../admin/script-utils/startcbinary/numactl.go | 72 -- .../script-utils/startcbinary/numactl_test.go | 89 --- .../admin/script-utils/startcbinary/progspec.go | 144 ---- .../script-utils/startcbinary/progspec_test.go | 38 - .../script-utils/startcbinary/startcbinary.go | 47 -- .../admin/script-utils/startcbinary/tuning.go | 13 - .../admin/script-utils/startcbinary/valgrind.go | 82 --- .../script-utils/startcbinary/valgrind_test.go | 92 --- .../admin/script-utils/startcbinary/vespamalloc.go | 57 -- .../admin/vespa-wrapper/configserver/check.go | 24 + .../admin/vespa-wrapper/configserver/env.go | 31 + .../configserver/fix_dirs_and_files.go | 27 + .../admin/vespa-wrapper/configserver/logd.go | 25 + .../admin/vespa-wrapper/configserver/runserver.go | 59 ++ .../admin/vespa-wrapper/configserver/start.go | 86 +++ .../admin/vespa-wrapper/configserver/zk.go | 22 + .../go/internal/admin/vespa-wrapper/logfmt/cmd.go | 46 ++ .../admin/vespa-wrapper/logfmt/formatflags.go | 41 ++ .../admin/vespa-wrapper/logfmt/formatflags_test.go | 30 + .../admin/vespa-wrapper/logfmt/handleline.go | 179 +++++ .../admin/vespa-wrapper/logfmt/internal.go | 106 +++ .../admin/vespa-wrapper/logfmt/internal_names.txt | 782 +++++++++++++++++++++ .../vespa-wrapper/logfmt/internal_notnames.txt | 8 + .../admin/vespa-wrapper/logfmt/internal_test.go | 34 + .../admin/vespa-wrapper/logfmt/levelflags.go | 72 ++ .../admin/vespa-wrapper/logfmt/levelflags_test.go | 74 ++ .../internal/admin/vespa-wrapper/logfmt/options.go | 48 ++ .../admin/vespa-wrapper/logfmt/plusminusflag.go | 67 ++ .../admin/vespa-wrapper/logfmt/regexflag.go | 38 + .../admin/vespa-wrapper/logfmt/regexflag_test.go | 41 ++ .../admin/vespa-wrapper/logfmt/runlogfmt.go | 85 +++ .../admin/vespa-wrapper/logfmt/showflags.go | 76 ++ .../admin/vespa-wrapper/logfmt/showflags_test.go | 61 ++ .../go/internal/admin/vespa-wrapper/logfmt/tail.go | 13 + .../admin/vespa-wrapper/logfmt/tail_not_unix.go | 15 + .../admin/vespa-wrapper/logfmt/tail_unix.go | 170 +++++ client/go/internal/admin/vespa-wrapper/main.go | 107 +++ .../admin/vespa-wrapper/services/configproxy.go | 143 ++++ .../internal/admin/vespa-wrapper/services/env.go | 25 + .../admin/vespa-wrapper/services/prechecks.go | 37 + .../admin/vespa-wrapper/services/sentinel.go | 93 +++ .../internal/admin/vespa-wrapper/services/start.go | 69 ++ .../internal/admin/vespa-wrapper/services/stop.go | 35 + .../admin/vespa-wrapper/services/tuning.go | 51 ++ .../admin/vespa-wrapper/standalone/start.go | 53 ++ .../admin/vespa-wrapper/startcbinary/cmd.go | 48 ++ .../admin/vespa-wrapper/startcbinary/common_env.go | 83 +++ .../vespa-wrapper/startcbinary/mockbin/bad-numactl | 25 + .../startcbinary/mockbin/good-numactl | 25 + .../startcbinary/mockbin/has-valgrind | 6 + .../vespa-wrapper/startcbinary/mockbin/no-numactl | 6 + .../vespa-wrapper/startcbinary/mockbin/no-valgrind | 6 + .../admin/vespa-wrapper/startcbinary/numactl.go | 72 ++ .../vespa-wrapper/startcbinary/numactl_test.go | 89 +++ .../admin/vespa-wrapper/startcbinary/progspec.go | 144 ++++ .../vespa-wrapper/startcbinary/progspec_test.go | 38 + .../vespa-wrapper/startcbinary/startcbinary.go | 47 ++ .../admin/vespa-wrapper/startcbinary/tuning.go | 13 + .../admin/vespa-wrapper/startcbinary/valgrind.go | 82 +++ .../vespa-wrapper/startcbinary/valgrind_test.go | 92 +++ .../vespa-wrapper/startcbinary/vespamalloc.go | 57 ++ client/go/internal/cli/cmd/vespa/main.go | 2 +- client/go/internal/vespa/find_home.go | 2 +- client/go/internal/vespa/switch_user.go | 2 +- 108 files changed, 3739 insertions(+), 3738 deletions(-) delete mode 100644 client/go/internal/admin/script-utils/configserver/check.go delete mode 100644 client/go/internal/admin/script-utils/configserver/env.go delete mode 100644 client/go/internal/admin/script-utils/configserver/fix_dirs_and_files.go delete mode 100644 client/go/internal/admin/script-utils/configserver/logd.go delete mode 100644 client/go/internal/admin/script-utils/configserver/runserver.go delete mode 100644 client/go/internal/admin/script-utils/configserver/start.go delete mode 100644 client/go/internal/admin/script-utils/configserver/zk.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/cmd.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/formatflags.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/formatflags_test.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/handleline.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/internal.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/internal_names.txt delete mode 100644 client/go/internal/admin/script-utils/logfmt/internal_notnames.txt delete mode 100644 client/go/internal/admin/script-utils/logfmt/internal_test.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/levelflags.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/levelflags_test.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/options.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/plusminusflag.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/regexflag.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/regexflag_test.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/runlogfmt.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/showflags.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/showflags_test.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/tail.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/tail_not_unix.go delete mode 100644 client/go/internal/admin/script-utils/logfmt/tail_unix.go delete mode 100644 client/go/internal/admin/script-utils/main.go delete mode 100644 client/go/internal/admin/script-utils/services/configproxy.go delete mode 100644 client/go/internal/admin/script-utils/services/env.go delete mode 100644 client/go/internal/admin/script-utils/services/prechecks.go delete mode 100644 client/go/internal/admin/script-utils/services/sentinel.go delete mode 100644 client/go/internal/admin/script-utils/services/start.go delete mode 100644 client/go/internal/admin/script-utils/services/stop.go delete mode 100644 client/go/internal/admin/script-utils/services/tuning.go delete mode 100644 client/go/internal/admin/script-utils/standalone/start.go delete mode 100644 client/go/internal/admin/script-utils/startcbinary/cmd.go delete mode 100644 client/go/internal/admin/script-utils/startcbinary/common_env.go delete mode 100755 client/go/internal/admin/script-utils/startcbinary/mockbin/bad-numactl delete mode 100755 client/go/internal/admin/script-utils/startcbinary/mockbin/good-numactl delete mode 100755 client/go/internal/admin/script-utils/startcbinary/mockbin/has-valgrind delete mode 100755 client/go/internal/admin/script-utils/startcbinary/mockbin/no-numactl delete mode 100755 client/go/internal/admin/script-utils/startcbinary/mockbin/no-valgrind delete mode 100644 client/go/internal/admin/script-utils/startcbinary/numactl.go delete mode 100644 client/go/internal/admin/script-utils/startcbinary/numactl_test.go delete mode 100644 client/go/internal/admin/script-utils/startcbinary/progspec.go delete mode 100644 client/go/internal/admin/script-utils/startcbinary/progspec_test.go delete mode 100644 client/go/internal/admin/script-utils/startcbinary/startcbinary.go delete mode 100644 client/go/internal/admin/script-utils/startcbinary/tuning.go delete mode 100644 client/go/internal/admin/script-utils/startcbinary/valgrind.go delete mode 100644 client/go/internal/admin/script-utils/startcbinary/valgrind_test.go delete mode 100644 client/go/internal/admin/script-utils/startcbinary/vespamalloc.go create mode 100644 client/go/internal/admin/vespa-wrapper/configserver/check.go create mode 100644 client/go/internal/admin/vespa-wrapper/configserver/env.go create mode 100644 client/go/internal/admin/vespa-wrapper/configserver/fix_dirs_and_files.go create mode 100644 client/go/internal/admin/vespa-wrapper/configserver/logd.go create mode 100644 client/go/internal/admin/vespa-wrapper/configserver/runserver.go create mode 100644 client/go/internal/admin/vespa-wrapper/configserver/start.go create mode 100644 client/go/internal/admin/vespa-wrapper/configserver/zk.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/cmd.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/formatflags.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/formatflags_test.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/handleline.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/internal.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/internal_names.txt create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/internal_notnames.txt create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/internal_test.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/levelflags.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/levelflags_test.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/options.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/plusminusflag.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/regexflag.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/regexflag_test.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/runlogfmt.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/showflags.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/showflags_test.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/tail.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/tail_not_unix.go create mode 100644 client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go create mode 100644 client/go/internal/admin/vespa-wrapper/main.go create mode 100644 client/go/internal/admin/vespa-wrapper/services/configproxy.go create mode 100644 client/go/internal/admin/vespa-wrapper/services/env.go create mode 100644 client/go/internal/admin/vespa-wrapper/services/prechecks.go create mode 100644 client/go/internal/admin/vespa-wrapper/services/sentinel.go create mode 100644 client/go/internal/admin/vespa-wrapper/services/start.go create mode 100644 client/go/internal/admin/vespa-wrapper/services/stop.go create mode 100644 client/go/internal/admin/vespa-wrapper/services/tuning.go create mode 100644 client/go/internal/admin/vespa-wrapper/standalone/start.go create mode 100644 client/go/internal/admin/vespa-wrapper/startcbinary/cmd.go create mode 100644 client/go/internal/admin/vespa-wrapper/startcbinary/common_env.go create mode 100755 client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/bad-numactl create mode 100755 client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/good-numactl create mode 100755 client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/has-valgrind create mode 100755 client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/no-numactl create mode 100755 client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/no-valgrind create mode 100644 client/go/internal/admin/vespa-wrapper/startcbinary/numactl.go create mode 100644 client/go/internal/admin/vespa-wrapper/startcbinary/numactl_test.go create mode 100644 client/go/internal/admin/vespa-wrapper/startcbinary/progspec.go create mode 100644 client/go/internal/admin/vespa-wrapper/startcbinary/progspec_test.go create mode 100644 client/go/internal/admin/vespa-wrapper/startcbinary/startcbinary.go create mode 100644 client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go create mode 100644 client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go create mode 100644 client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go create mode 100644 client/go/internal/admin/vespa-wrapper/startcbinary/vespamalloc.go (limited to 'client') diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index a029af876f0..b73f4e153ff 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -4,37 +4,37 @@ set(GODIR ${CMAKE_CURRENT_SOURCE_DIR}/go) file(GLOB_RECURSE GOSRCFILES ${GODIR}/*.go) -add_custom_command(OUTPUT ${GODIR}/bin/vespa ${GODIR}/bin/script-utils +add_custom_command(OUTPUT ${GODIR}/bin/vespa ${GODIR}/bin/vespa-wrapper COMMAND make DEPENDS ${GODIR}/Makefile ${GOSRCFILES} WORKING_DIRECTORY ${GODIR}) -add_custom_target(client_go_binaries ALL DEPENDS ${GODIR}/bin/vespa ${GODIR}/bin/script-utils) - -install(PROGRAMS ${GODIR}/bin/script-utils DESTINATION libexec/vespa) - -install_symlink(libexec/vespa/script-utils bin/vespa-logfmt) -install_symlink(libexec/vespa/script-utils bin/vespa-deploy) -install_symlink(libexec/vespa/script-utils bin/vespa-security-env) -install_symlink(libexec/vespa/script-utils bin/vespa-get-cluster-state) -install_symlink(libexec/vespa/script-utils bin/vespa-get-node-state) -install_symlink(libexec/vespa/script-utils bin/vespa-set-node-state) -install_symlink(libexec/vespa/script-utils bin/vespa-start-configserver) -install_symlink(libexec/vespa/script-utils bin/vespa-start-services bin) -install_symlink(libexec/vespa/script-utils bin/vespa-stop-services bin) - -install_symlink(libexec/vespa/script-utils bin/vespa-get-config) -install_symlink(libexec/vespa/script-utils bin/vespa-verify-ranksetup) -install_symlink(libexec/vespa/script-utils bin/vespa-config-status) -install_symlink(libexec/vespa/script-utils bin/vespa-configproxy-cmd) -install_symlink(libexec/vespa/script-utils bin/vespa-doclocator) -install_symlink(libexec/vespa/script-utils bin/vespa-model-inspect) -install_symlink(libexec/vespa/script-utils bin/vespa-proton-cmd) -install_symlink(libexec/vespa/script-utils bin/vespa-rpc-invoke) -install_symlink(libexec/vespa/script-utils bin/vespa-sentinel-cmd) -install_symlink(libexec/vespa/script-utils bin/vespa-route) -install_symlink(libexec/vespa/script-utils bin/vespa-transactionlog-inspect) - -install_symlink(libexec/vespa/script-utils sbin/vespa-distributord) -install_symlink(libexec/vespa/script-utils sbin/vespa-proton) -install_symlink(libexec/vespa/script-utils sbin/vespa-storaged) +add_custom_target(client_go_binaries ALL DEPENDS ${GODIR}/bin/vespa ${GODIR}/bin/vespa-wrapper) + +install(PROGRAMS ${GODIR}/bin/vespa-wrapper DESTINATION libexec/vespa) + +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-logfmt) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-deploy) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-security-env) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-get-cluster-state) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-get-node-state) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-set-node-state) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-start-configserver) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-start-services bin) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-stop-services bin) + +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-get-config) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-verify-ranksetup) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-config-status) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-configproxy-cmd) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-doclocator) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-model-inspect) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-proton-cmd) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-rpc-invoke) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-sentinel-cmd) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-route) +install_symlink(libexec/vespa/vespa-wrapper bin/vespa-transactionlog-inspect) + +install_symlink(libexec/vespa/vespa-wrapper sbin/vespa-distributord) +install_symlink(libexec/vespa/vespa-wrapper sbin/vespa-proton) +install_symlink(libexec/vespa/vespa-wrapper sbin/vespa-storaged) diff --git a/client/go/internal/admin/script-utils/configserver/check.go b/client/go/internal/admin/script-utils/configserver/check.go deleted file mode 100644 index a0248dd128f..00000000000 --- a/client/go/internal/admin/script-utils/configserver/check.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package configserver - -import ( - "fmt" - - "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -func checkIsConfigserver(myname string) { - onlyHosts := defaults.VespaConfigserverHosts() - for _, hn := range onlyHosts { - if hn == "localhost" || hn == myname { - trace.Debug("should run configserver:", hn) - return - } - } - trace.Warning("only these hosts should run a config server:", onlyHosts) - util.JustExitMsg(fmt.Sprintf("this host [%s] should not run a config server", myname)) -} diff --git a/client/go/internal/admin/script-utils/configserver/env.go b/client/go/internal/admin/script-utils/configserver/env.go deleted file mode 100644 index 1c4c33f5628..00000000000 --- a/client/go/internal/admin/script-utils/configserver/env.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package configserver - -import ( - "fmt" - "os" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -func exportSettings(vespaHome string) { - vlt := fmt.Sprintf("file:%s/logs/vespa/vespa.log", vespaHome) - lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome) - lcf := fmt.Sprintf("%s/configserver.logcontrol", lcd) - dlp := fmt.Sprintf("%s/lib64", vespaHome) - app := fmt.Sprintf("%s/conf/configserver-app", vespaHome) - os.Setenv(envvars.VESPA_LOG_TARGET, vlt) - os.Setenv(envvars.VESPA_LOG_CONTROL_DIR, lcd) - os.Setenv(envvars.VESPA_LOG_CONTROL_FILE, lcf) - os.Setenv(envvars.VESPA_SERVICE_NAME, "configserver") - os.Setenv(envvars.LD_LIBRARY_PATH, dlp) - os.Setenv(envvars.JAVAVM_LD_PRELOAD, "") - os.Setenv(envvars.LD_PRELOAD, "") - os.Setenv(envvars.STANDALONE_JDISC_APP_LOCATION, app) - os.Setenv(envvars.STANDALONE_JDISC_DEPLOYMENT_PROFILE, "configserver") - os.Setenv(envvars.MALLOC_ARENA_MAX, "1") - util.OptionallyReduceTimerFrequency() -} diff --git a/client/go/internal/admin/script-utils/configserver/fix_dirs_and_files.go b/client/go/internal/admin/script-utils/configserver/fix_dirs_and_files.go deleted file mode 100644 index 8410d3f657c..00000000000 --- a/client/go/internal/admin/script-utils/configserver/fix_dirs_and_files.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package configserver - -import ( - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func makeFixSpec() util.FixSpec { - vespaUid, vespaGid := vespa.FindVespaUidAndGid() - return util.FixSpec{ - UserId: vespaUid, - GroupId: vespaGid, - DirMode: 0755, - FileMode: 0644, - } -} - -func fixDirsAndFiles(fixSpec util.FixSpec) { - fixSpec.FixDir("var/zookeeper") - fixSpec.FixDir("var/zookeeper/conf") - fixSpec.FixDir("var/zookeeper/version-2") - fixSpec.FixFile("var/zookeeper/conf/zookeeper.cfg") - fixSpec.FixFile("var/zookeeper/myid") -} diff --git a/client/go/internal/admin/script-utils/configserver/logd.go b/client/go/internal/admin/script-utils/configserver/logd.go deleted file mode 100644 index 39a2100b20f..00000000000 --- a/client/go/internal/admin/script-utils/configserver/logd.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package configserver - -import ( - "os" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -func maybeStartLogd() { - v1 := os.Getenv(envvars.CLOUDCONFIG_SERVER_MULTITENANT) - v2 := os.Getenv(envvars.VESPA_CONFIGSERVER_MULTITENANT) - if v1 == "true" || v2 == "true" { - backticks := util.BackTicksForwardStderr - out, err := backticks.Run("libexec/vespa/start-logd") - if err != nil { - panic(err) - } - trace.Info(out) - } -} diff --git a/client/go/internal/admin/script-utils/configserver/runserver.go b/client/go/internal/admin/script-utils/configserver/runserver.go deleted file mode 100644 index bef50009d8d..00000000000 --- a/client/go/internal/admin/script-utils/configserver/runserver.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package configserver - -import ( - "fmt" - - "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -const ( - PROG_NAME = "vespa-runserver" -) - -type RunServer struct { - ServiceName string - Args []string -} - -func (rs *RunServer) PidFile() string { - varRunDir := defaults.UnderVespaHome("var/run") - return fmt.Sprintf("%s/%s.pid", varRunDir, rs.ServiceName) -} - -func (rs *RunServer) ProgPath() string { - p := fmt.Sprintf("%s/bin64/%s", defaults.VespaHome(), PROG_NAME) - if util.IsExecutableFile(p) { - return p - } - p = fmt.Sprintf("%s/bin/%s", defaults.VespaHome(), PROG_NAME) - if util.IsExecutableFile(p) { - return p - } - panic(fmt.Errorf("not an executable file: %s", p)) -} - -func (rs *RunServer) WouldRun() bool { - backticks := util.BackTicksForwardStderr - out, err := backticks.Run(rs.ProgPath(), "-s", rs.ServiceName, "-p", rs.PidFile(), "-W") - trace.Trace("output from -W:", out, "error:", err) - return err == nil -} - -func (rs *RunServer) Exec(prog string) { - argv := util.ArrayList[string]{ - PROG_NAME, - "-s", rs.ServiceName, - "-r", "30", - "-p", rs.PidFile(), - "--", - prog, - } - argv.AppendAll(rs.Args...) - err := util.Execvp(rs.ProgPath(), argv) - util.JustExitWith(err) -} diff --git a/client/go/internal/admin/script-utils/configserver/start.go b/client/go/internal/admin/script-utils/configserver/start.go deleted file mode 100644 index 8ea3e9603a2..00000000000 --- a/client/go/internal/admin/script-utils/configserver/start.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package configserver - -import ( - "os" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -const ( - SERVICE_NAME = "configserver" -) - -func commonPreChecks() (veHome string) { - if doTrace := os.Getenv(envvars.TRACE_JVM_STARTUP); doTrace != "" { - trace.AdjustVerbosity(1) - } - if doDebug := os.Getenv(envvars.DEBUG_JVM_STARTUP); doDebug != "" { - trace.AdjustVerbosity(2) - } - _ = vespa.FindAndVerifyVespaHome() - err := vespa.LoadDefaultEnv() - if err != nil { - panic(err) - } - veHome = vespa.FindAndVerifyVespaHome() - veHost, e := vespa.FindOurHostname() - if e != nil { - trace.Warning("could not detect hostname:", err, "; using fallback:", veHost) - } - checkIsConfigserver(veHost) - e = os.Chdir(veHome) - if e != nil { - util.JustExitWith(e) - } - return -} - -func JustStartConfigserver() int { - vespaHome := commonPreChecks() - vespa.CheckCorrectUser() - util.TuneResourceLimits() - util.TuneLogging(SERVICE_NAME, "com.google.api.client.http.HttpTransport", "config=off") - exportSettings(vespaHome) - removeStaleZkLocks(vespaHome) - c := jvm.NewStandaloneContainer(SERVICE_NAME) - jvmOpts := c.JvmOptions() - if extra := os.Getenv(envvars.VESPA_CONFIGSERVER_JVMARGS); extra != "" { - jvmOpts.AddJvmArgsFromString(extra) - } - minFallback := jvm.MegaBytesOfMemory(128) - maxFallback := jvm.MegaBytesOfMemory(2048) - jvmOpts.AddDefaultHeapSizeArgs(minFallback, maxFallback) - c.Exec() - // unreachable: - return 1 -} - -func runConfigserverWithRunserver() int { - commonPreChecks() - vespa.CheckCorrectUser() - rs := RunServer{ - ServiceName: SERVICE_NAME, - Args: []string{"just-start-configserver"}, - } - rs.Exec("libexec/vespa/script-utils") - return 1 -} - -func StartConfigserverEtc() int { - vespaHome := commonPreChecks() - vespa.RunPreStart() - util.TuneResourceLimits() - fixSpec := makeFixSpec() - fixDirsAndFiles(fixSpec) - exportSettings(vespaHome) - vespa.MaybeSwitchUser("vespa-start-configserver") - maybeStartLogd() - return runConfigserverWithRunserver() -} diff --git a/client/go/internal/admin/script-utils/configserver/zk.go b/client/go/internal/admin/script-utils/configserver/zk.go deleted file mode 100644 index 91ddf522848..00000000000 --- a/client/go/internal/admin/script-utils/configserver/zk.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package configserver - -import ( - "fmt" - - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -const ( - ZOOKEEPER_LOG_FILE_PREFIX = "logs/vespa/zookeeper.configserver" -) - -func removeStaleZkLocks(vespaHome string) { - backticks := util.BackTicksIgnoreStderr - cmd := fmt.Sprintf("rm -f '%s/%s'*lck", vespaHome, ZOOKEEPER_LOG_FILE_PREFIX) - trace.Trace("cleaning locks:", cmd) - backticks.Run("/bin/sh", "-c", cmd) -} diff --git a/client/go/internal/admin/script-utils/logfmt/cmd.go b/client/go/internal/admin/script-utils/logfmt/cmd.go deleted file mode 100644 index a8675c37356..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/cmd.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa-logfmt command -// Author: arnej - -package logfmt - -import ( - "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/internal/cli/build" -) - -func NewLogfmtCmd() *cobra.Command { - var ( - curOptions Options = NewOptions() - ) - cmd := &cobra.Command{ - Use: "vespa-logfmt", - Short: "convert vespa.log to human-readable format", - Long: `vespa-logfmt takes input in the internal vespa format -and converts it to something human-readable`, - Version: build.Version, - Run: func(cmd *cobra.Command, args []string) { - RunLogfmt(&curOptions, args) - }, - } - cmd.Flags().VarP(&curOptions.ShowLevels, "level", "l", "turn levels on/off\n") - cmd.Flags().VarP(&curOptions.ShowFields, "show", "s", "turn fields shown on/off\n") - cmd.Flags().VarP(&curOptions.ComponentFilter, "component", "c", "select components by regexp") - cmd.Flags().VarP(&curOptions.MessageFilter, "message", "m", "select messages by regexp") - cmd.Flags().BoolVarP(&curOptions.OnlyInternal, "internal", "i", false, "select only internal components") - cmd.Flags().BoolVar(&curOptions.TruncateService, "truncateservice", false, "truncate service name") - cmd.Flags().BoolVar(&curOptions.TruncateService, "ts", false, "") - cmd.Flags().BoolVarP(&curOptions.FollowTail, "follow", "f", false, "follow logfile with tail -f") - cmd.Flags().BoolVarP(&curOptions.DequoteNewlines, "nldequote", "N", false, "dequote newlines embedded in message") - cmd.Flags().BoolVarP(&curOptions.DequoteNewlines, "dequotenewlines", "n", false, "dequote newlines embedded in message") - cmd.Flags().BoolVarP(&curOptions.TruncateComponent, "truncatecomponent", "t", false, "truncate component name") - cmd.Flags().BoolVar(&curOptions.TruncateComponent, "tc", false, "") - cmd.Flags().StringVarP(&curOptions.OnlyHostname, "host", "H", "", "select only one host") - cmd.Flags().StringVarP(&curOptions.OnlyPid, "pid", "p", "", "select only one process ID") - cmd.Flags().StringVarP(&curOptions.OnlyService, "service", "S", "", "select only one service") - cmd.Flags().VarP(&curOptions.Format, "format", "F", "select logfmt output format, vespa (default), json or raw are supported. The json output format is not stable, and will change in the future.") - cmd.Flags().MarkHidden("tc") - cmd.Flags().MarkHidden("ts") - cmd.Flags().MarkHidden("dequotenewlines") - return cmd -} diff --git a/client/go/internal/admin/script-utils/logfmt/formatflags.go b/client/go/internal/admin/script-utils/logfmt/formatflags.go deleted file mode 100644 index 097746d696f..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/formatflags.go +++ /dev/null @@ -1,41 +0,0 @@ -package logfmt - -import ( - "fmt" - "strings" -) - -type OutputFormat int - -const ( - FormatVespa OutputFormat = iota //default is vespa - FormatRaw - FormatJSON -) - -func (v *OutputFormat) Type() string { - return "output format" -} - -func (v *OutputFormat) String() string { - flagNames := []string{ - "vespa", - "raw", - "json", - } - return flagNames[*v] -} - -func (v *OutputFormat) Set(val string) error { - switch strings.ToLower(val) { - case "vespa": - *v = FormatVespa - case "raw": - *v = FormatRaw - case "json": - *v = FormatJSON - default: - return fmt.Errorf("'%s' is not a valid format argument", val) - } - return nil -} diff --git a/client/go/internal/admin/script-utils/logfmt/formatflags_test.go b/client/go/internal/admin/script-utils/logfmt/formatflags_test.go deleted file mode 100644 index 53c47d24208..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/formatflags_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package logfmt - -import ( - "fmt" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestOutputFormat(t *testing.T) { - type args struct { - val string - } - tests := []struct { - expected OutputFormat - arg string - wantErr assert.ErrorAssertionFunc - }{ - {FormatVespa, "vespa", assert.NoError}, - {FormatRaw, "raw", assert.NoError}, - {FormatJSON, "json", assert.NoError}, - {-1, "foo", assert.Error}, - } - for _, tt := range tests { - t.Run(tt.arg, func(t *testing.T) { - var v OutputFormat = -1 - tt.wantErr(t, v.Set(tt.arg), fmt.Sprintf("Set(%v)", tt.arg)) - assert.Equal(t, v, tt.expected) - }) - } -} diff --git a/client/go/internal/admin/script-utils/logfmt/handleline.go b/client/go/internal/admin/script-utils/logfmt/handleline.go deleted file mode 100644 index 813ca82acb4..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/handleline.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -package logfmt - -import ( - "bytes" - "encoding/json" - "fmt" - "strconv" - "strings" - "time" -) - -type logFields struct { - timestamp string // seconds, optional fractional seconds - host string - pid string // pid, optional tid - service string - component string - level string - messages []string -} - -// handle a line in "vespa.log" format; do filtering and formatting as specified in opts -func handleLine(opts *Options, line string) (string, error) { - fieldStrings := strings.SplitN(line, "\t", 7) - if len(fieldStrings) < 7 { - return "", fmt.Errorf("not enough fields: '%s'", line) - } - fields := logFields{ - timestamp: fieldStrings[0], - host: fieldStrings[1], - pid: fieldStrings[2], - service: fieldStrings[3], - component: fieldStrings[4], - level: fieldStrings[5], - messages: fieldStrings[6:], - } - - if !opts.showLevel(fields.level) { - return "", nil - } - if opts.OnlyHostname != "" && opts.OnlyHostname != fields.host { - return "", nil - } - if opts.OnlyPid != "" && opts.OnlyPid != fields.pid { - return "", nil - } - if opts.OnlyService != "" && opts.OnlyService != fields.service { - return "", nil - } - if opts.OnlyInternal && !isInternal(fields.component) { - return "", nil - } - if opts.ComponentFilter.unmatched(fields.component) { - return "", nil - } - if opts.MessageFilter.unmatched(strings.Join(fields.messages, "\t")) { - return "", nil - } - - switch opts.Format { - case FormatRaw: - return line + "\n", nil - case FormatJSON: - return handleLineJson(opts, &fields) - case FormatVespa: - fallthrough - default: - return handleLineVespa(opts, &fields) - } -} - -func parseTimestamp(timestamp string) (time.Time, error) { - secs, err := strconv.ParseFloat(timestamp, 64) - if err != nil { - return time.Time{}, err - } - nsecs := int64(secs * 1e9) - return time.Unix(0, nsecs), nil -} - -type logFieldsJson struct { - Timestamp string `json:"timestamp"` - Host string `json:"host"` - Pid string `json:"pid"` - Service string `json:"service"` - Component string `json:"component"` - Level string `json:"level"` - Messages []string `json:"messages"` -} - -func handleLineJson(_ *Options, fields *logFields) (string, error) { - timestamp, err := parseTimestamp(fields.timestamp) - if err != nil { - return "", err - } - outputFields := logFieldsJson{ - Timestamp: timestamp.Format(time.RFC3339Nano), - Host: fields.host, - Pid: fields.pid, - Service: fields.service, - Component: fields.component, - Level: fields.level, - Messages: fields.messages, - } - buf := bytes.Buffer{} - if err := json.NewEncoder(&buf).Encode(&outputFields); err != nil { - return "", err - } - return buf.String(), nil -} - -func handleLineVespa(opts *Options, fields *logFields) (string, error) { - var buf strings.Builder - - if opts.showField("fmttime") { - timestamp, err := parseTimestamp(fields.timestamp) - if err != nil { - return "", err - } - if opts.showField("usecs") { - buf.WriteString(timestamp.Format("[2006-01-02 15:04:05.000000] ")) - } else if opts.showField("msecs") { - buf.WriteString(timestamp.Format("[2006-01-02 15:04:05.000] ")) - } else { - buf.WriteString(timestamp.Format("[2006-01-02 15:04:05] ")) - } - } else if opts.showField("time") { - buf.WriteString(fields.timestamp) - buf.WriteString(" ") - } - if opts.showField("host") { - buf.WriteString(fmt.Sprintf("%-8s ", fields.host)) - } - if opts.showField("level") { - buf.WriteString(fmt.Sprintf("%-7s ", strings.ToUpper(fields.level))) - } - if opts.showField("pid") { - // OnlyPid, _, _ := strings.Cut(pidfield, "/") - buf.WriteString(fmt.Sprintf("%6s ", fields.pid)) - } - if opts.showField("service") { - if opts.TruncateService { - buf.WriteString(fmt.Sprintf("%-9.9s ", fields.service)) - } else { - buf.WriteString(fmt.Sprintf("%-16s ", fields.service)) - } - } - if opts.showField("component") { - if opts.TruncateComponent { - buf.WriteString(fmt.Sprintf("%-15.15s ", fields.component)) - } else { - buf.WriteString(fmt.Sprintf("%s\t", fields.component)) - } - } - if opts.showField("message") { - var msgBuf strings.Builder - for idx, message := range fields.messages { - if idx > 0 { - msgBuf.WriteString("\n\t") - } - if opts.DequoteNewlines { - message = strings.ReplaceAll(message, "\\n\\t", "\n\t") - message = strings.ReplaceAll(message, "\\n", "\n\t") - } - msgBuf.WriteString(message) - } - message := msgBuf.String() - if strings.Contains(message, "\n") { - buf.WriteString("\n\t") - } - buf.WriteString(message) - } - buf.WriteString("\n") - return buf.String(), nil -} diff --git a/client/go/internal/admin/script-utils/logfmt/internal.go b/client/go/internal/admin/script-utils/logfmt/internal.go deleted file mode 100644 index 992c537f939..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/internal.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -package logfmt - -import ( - "strings" -) - -// is componentName a vespa-internal name? - -func isInternal(componentName string) bool { - cs := strings.Split(componentName, ".") - if len(cs) == 0 || cs[0] != "Container" { - return true - } - if len(cs) < 3 { - return false - } - if cs[1] == "ai" && cs[2] == "vespa" { - return true - } - if cs[1] == "com" && cs[2] == "yahoo" && len(cs) > 3 { - return internalComYahooNames[cs[3]] - } - return false -} - -// a constant: -var internalComYahooNames = map[string]bool{ - "abicheck": true, - "api": true, - "application": true, - "binaryprefix": true, - "clientmetrics": true, - "cloud": true, - "collections": true, - "component": true, - "compress": true, - "concurrent": true, - "configtest": true, - "config": true, - "container": true, - "data": true, - "docprocs": true, - "docproc": true, - "documentapi": true, - "documentmodel": true, - "document": true, - "dummyreceiver": true, - "embedding": true, - "errorhandling": true, - "exception": true, - "feedapi": true, - "feedhandler": true, - "filedistribution": true, - "fs4": true, - "fsa": true, - "geo": true, - "io": true, - "javacc": true, - "jdisc": true, - "jrt": true, - "lang": true, - "language": true, - "logserver": true, - "log": true, - "messagebus": true, - "metrics": true, - "nativec": true, - "net": true, - "osgi": true, - "path": true, - "plugin": true, - "prelude": true, - "processing": true, - "protect": true, - "reflection": true, - "restapi": true, - "schema": true, - "searchdefinition": true, - "searchlib": true, - "search": true, - "security": true, - "slime": true, - "socket": true, - "statistics": true, - "stream": true, - "system": true, - "tensor": true, - "test": true, - "text": true, - "time": true, - "transaction": true, - "vdslib": true, - "vespaclient": true, - "vespafeeder": true, - "vespaget": true, - "vespastat": true, - "vespasummarybenchmark": true, - "vespa": true, - "vespavisit": true, - "vespaxmlparser": true, - "yolean": true, -} diff --git a/client/go/internal/admin/script-utils/logfmt/internal_names.txt b/client/go/internal/admin/script-utils/logfmt/internal_names.txt deleted file mode 100644 index cc554546fcc..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/internal_names.txt +++ /dev/null @@ -1,782 +0,0 @@ -sentinel.sentinel.config -searchnode.eval.Test -Container.ai.vespa.client.dsl.Test -Container.ai.vespa.cloud.Test -Container.ai.vespa.embedding.Test -Container.ai.vespa.explicitversion.Test -Container.ai.vespa.explicitversion_dep.Test -Container.ai.vespa.feed.client.Test -Container.ai.vespa.feed.client.impl.Test -Container.ai.vespa.hosted.api.Test -Container.ai.vespa.hosted.cd.Test -Container.ai.vespa.hosted.cd.cloud.impl.Test -Container.ai.vespa.hosted.cd.commons.Test -Container.ai.vespa.hosted.cd.internal.Test -Container.ai.vespa.hosted.client.Test -Container.ai.vespa.hosted.plugin.Test -Container.ai.vespa.http.Test -Container.ai.vespa.intellij.schema.Test -Container.ai.vespa.intellij.schema.findUsages.Test -Container.ai.vespa.intellij.schema.hierarchy.Test -Container.ai.vespa.intellij.schema.lexer.Test -Container.ai.vespa.intellij.schema.model.Test -Container.ai.vespa.intellij.schema.parser.Test -Container.ai.vespa.intellij.schema.psi.Test -Container.ai.vespa.intellij.schema.psi.impl.Test -Container.ai.vespa.intellij.schema.structure.Test -Container.ai.vespa.intellij.schema.utils.Test -Container.ai.vespa.logserver.protocol.Test -Container.ai.vespa.metricsproxy.core.Test -Container.ai.vespa.metricsproxy.http.Test -Container.ai.vespa.metricsproxy.http.application.Test -Container.ai.vespa.metricsproxy.http.metrics.Test -Container.ai.vespa.metricsproxy.http.prometheus.Test -Container.ai.vespa.metricsproxy.http.yamas.Test -Container.ai.vespa.metricsproxy.metric.Test -Container.ai.vespa.metricsproxy.metric.dimensions.Test -Container.ai.vespa.metricsproxy.metric.model.Test -Container.ai.vespa.metricsproxy.metric.model.json.Test -Container.ai.vespa.metricsproxy.metric.model.processing.Test -Container.ai.vespa.metricsproxy.metric.model.prometheus.Test -Container.ai.vespa.metricsproxy.node.Test -Container.ai.vespa.metricsproxy.rpc.Test -Container.ai.vespa.metricsproxy.service.Test -Container.ai.vespa.metricsproxy.telegraf.Test -Container.ai.vespa.modelintegration.evaluator.Test -Container.ai.vespa.models.evaluation.Test -Container.ai.vespa.models.handler.Test -Container.ai.vespa.noversion.Test -Container.ai.vespa.noversion_dep.Test -Container.ai.vespa.rankingexpression.importer.Test -Container.ai.vespa.rankingexpression.importer.configmodelview.Test -Container.ai.vespa.rankingexpression.importer.lightgbm.Test -Container.ai.vespa.rankingexpression.importer.onnx.Test -Container.ai.vespa.rankingexpression.importer.operations.Test -Container.ai.vespa.rankingexpression.importer.tensorflow.Test -Container.ai.vespa.rankingexpression.importer.vespa.Test -Container.ai.vespa.rankingexpression.importer.vespa.parser.Test -Container.ai.vespa.rankingexpression.importer.xgboost.Test -Container.ai.vespa.reindexing.Test -Container.ai.vespa.reindexing.http.Test -Container.ai.vespa.searchlib.searchprotocol.protobuf.Test -Container.ai.vespa.util.http.hc4.Test -Container.ai.vespa.util.http.hc4.retry.Test -Container.ai.vespa.util.http.hc5.Test -Container.ai.vespa.validation.Test -Container.com.yahoo.abicheck.classtree.Test -Container.com.yahoo.abicheck.collector.Test -Container.com.yahoo.abicheck.mojo.Test -Container.com.yahoo.abicheck.setmatcher.Test -Container.com.yahoo.abicheck.signature.Test -Container.com.yahoo.api.annotations.Test -Container.com.yahoo.application.Test -Container.com.yahoo.application.container.Test -Container.com.yahoo.application.container.handler.Test -Container.com.yahoo.application.container.impl.Test -Container.com.yahoo.application.content.Test -Container.com.yahoo.application.preprocessor.Test -Container.com.yahoo.binaryprefix.Test -Container.com.yahoo.clientmetrics.Test -Container.com.yahoo.cloud.config.Test -Container.com.yahoo.collections.Test -Container.com.yahoo.component.Test -Container.com.yahoo.component.annotation.Test -Container.com.yahoo.component.chain.Test -Container.com.yahoo.component.chain.dependencies.Test -Container.com.yahoo.component.chain.dependencies.ordering.Test -Container.com.yahoo.component.chain.model.Test -Container.com.yahoo.component.provider.Test -Container.com.yahoo.compress.Test -Container.com.yahoo.concurrent.Test -Container.com.yahoo.concurrent.classlock.Test -Container.com.yahoo.concurrent.maintenance.Test -Container.com.yahoo.config.Test -Container.com.yahoo.config.application.Test -Container.com.yahoo.config.application.api.Test -Container.com.yahoo.config.application.api.xml.Test -Container.com.yahoo.config.codegen.Test -Container.com.yahoo.config.docproc.Test -Container.com.yahoo.config.ini.Test -Container.com.yahoo.config.model.Test -Container.com.yahoo.config.model.admin.Test -Container.com.yahoo.config.model.api.Test -Container.com.yahoo.config.model.api.container.Test -Container.com.yahoo.config.model.application.Test -Container.com.yahoo.config.model.application.provider.Test -Container.com.yahoo.config.model.builder.xml.Test -Container.com.yahoo.config.model.deploy.Test -Container.com.yahoo.config.model.graph.Test -Container.com.yahoo.config.model.producer.Test -Container.com.yahoo.config.model.provision.Test -Container.com.yahoo.config.model.test.Test -Container.com.yahoo.config.provision.Test -Container.com.yahoo.config.provision.exception.Test -Container.com.yahoo.config.provision.host.Test -Container.com.yahoo.config.provisioning.Test -Container.com.yahoo.config.provision.security.Test -Container.com.yahoo.config.provision.serialization.Test -Container.com.yahoo.config.provision.zone.Test -Container.com.yahoo.config.subscription.Test -Container.com.yahoo.config.subscription.impl.Test -Container.com.yahoo.configtest.Test -Container.com.yahoo.config.text.Test -Container.com.yahoo.container.Test -Container.com.yahoo.container.bundle.Test -Container.com.yahoo.container.core.Test -Container.com.yahoo.container.core.config.Test -Container.com.yahoo.container.core.config.testutil.Test -Container.com.yahoo.container.core.document.Test -Container.com.yahoo.container.core.documentapi.Test -Container.com.yahoo.container.core.http.Test -Container.com.yahoo.container.core.identity.Test -Container.com.yahoo.container.di.Test -Container.com.yahoo.container.di.componentgraph.Test -Container.com.yahoo.container.di.componentgraph.core.Test -Container.com.yahoo.container.di.componentgraph.cycle.Test -Container.com.yahoo.container.di.config.Test -Container.com.yahoo.container.handler.Test -Container.com.yahoo.container.handler.metrics.Test -Container.com.yahoo.container.handler.observability.Test -Container.com.yahoo.container.handler.test.Test -Container.com.yahoo.container.handler.threadpool.Test -Container.com.yahoo.container.http.Test -Container.com.yahoo.container.http.filter.Test -Container.com.yahoo.container.jdisc.Test -Container.com.yahoo.container.jdisc.athenz.Test -Container.com.yahoo.container.jdisc.component.Test -Container.com.yahoo.container.jdisc.config.Test -Container.com.yahoo.container.jdisc.messagebus.Test -Container.com.yahoo.container.jdisc.metric.Test -Container.com.yahoo.container.jdisc.metric.state.Test -Container.com.yahoo.container.jdisc.secretstore.Test -Container.com.yahoo.container.jdisc.state.Test -Container.com.yahoo.container.jdisc.utils.Test -Container.com.yahoo.container.logging.Test -Container.com.yahoo.container.plugin.bundle.Test -Container.com.yahoo.container.plugin.classanalysis.Test -Container.com.yahoo.container.plugin.mojo.Test -Container.com.yahoo.container.plugin.osgi.Test -Container.com.yahoo.container.plugin.util.Test -Container.com.yahoo.container.protect.Test -Container.com.yahoo.container.standalone.Test -Container.com.yahoo.container.usability.Test -Container.com.yahoo.data.Test -Container.com.yahoo.data.access.Test -Container.com.yahoo.data.access.simple.Test -Container.com.yahoo.data.access.slime.Test -Container.com.yahoo.docproc.Test -Container.com.yahoo.docproc.impl.Test -Container.com.yahoo.docproc.jdisc.Test -Container.com.yahoo.docproc.jdisc.messagebus.Test -Container.com.yahoo.docproc.jdisc.metric.Test -Container.com.yahoo.docproc.jdisc.observability.Test -Container.com.yahoo.docproc.proxy.Test -Container.com.yahoo.docprocs.indexing.Test -Container.com.yahoo.document.Test -Container.com.yahoo.document.annotation.Test -Container.com.yahoo.documentapi.Test -Container.com.yahoo.documentapi.local.Test -Container.com.yahoo.documentapi.messagebus.Test -Container.com.yahoo.documentapi.messagebus.protocol.Test -Container.com.yahoo.documentapi.messagebus.systemstate.rule.Test -Container.com.yahoo.documentapi.metrics.Test -Container.com.yahoo.document.config.Test -Container.com.yahoo.document.datatypes.Test -Container.com.yahoo.document.fieldpathupdate.Test -Container.com.yahoo.document.fieldset.Test -Container.com.yahoo.document.idstring.Test -Container.com.yahoo.document.internal.Test -Container.com.yahoo.document.json.Test -Container.com.yahoo.document.json.document.Test -Container.com.yahoo.document.json.readers.Test -Container.com.yahoo.documentmodel.Test -Container.com.yahoo.document.predicate.Test -Container.com.yahoo.document.restapi.resource.Test -Container.com.yahoo.document.select.Test -Container.com.yahoo.document.select.convert.Test -Container.com.yahoo.document.select.parser.Test -Container.com.yahoo.document.select.rule.Test -Container.com.yahoo.document.select.simple.Test -Container.com.yahoo.document.serialization.Test -Container.com.yahoo.document.update.Test -Container.com.yahoo.dummyreceiver.Test -Container.com.yahoo.embedding.Test -Container.com.yahoo.errorhandling.Test -Container.com.yahoo.exception.Test -Container.com.yahoo.feedapi.Test -Container.com.yahoo.feedhandler.Test -Container.com.yahoo.filedistribution.fileacquirer.Test -Container.com.yahoo.fs4.Test -Container.com.yahoo.fsa.Test -Container.com.yahoo.fsa.conceptnet.Test -Container.com.yahoo.fsa.segmenter.Test -Container.com.yahoo.fsa.topicpredictor.Test -Container.com.yahoo.geo.Test -Container.com.yahoo.io.Test -Container.com.yahoo.io.reader.Test -Container.com.yahoo.javacc.Test -Container.com.yahoo.jdisc.Test -Container.com.yahoo.jdisc.application.Test -Container.com.yahoo.jdisc.bundle.Test -Container.com.yahoo.jdisc.bundle.a.Test -Container.com.yahoo.jdisc.bundle.b.Test -Container.com.yahoo.jdisc.bundle.c.Test -Container.com.yahoo.jdisc.bundle.d.Test -Container.com.yahoo.jdisc.bundle.e.Test -Container.com.yahoo.jdisc.bundle.f.Test -Container.com.yahoo.jdisc.bundle.g.Test -Container.com.yahoo.jdisc.bundle.g_act.Test -Container.com.yahoo.jdisc.bundle.h.Test -Container.com.yahoo.jdisc.bundle.i.Test -Container.com.yahoo.jdisc.bundle.j.Test -Container.com.yahoo.jdisc.bundle.k.Test -Container.com.yahoo.jdisc.bundle.l.Test -Container.com.yahoo.jdisc.bundle.m.Test -Container.com.yahoo.jdisc.bundle.my_act.Test -Container.com.yahoo.jdisc.bundle.n.Test -Container.com.yahoo.jdisc.bundle.o.Test -Container.com.yahoo.jdisc.bundle.p.Test -Container.com.yahoo.jdisc.bundle.q.Test -Container.com.yahoo.jdisc.bundle.r.Test -Container.com.yahoo.jdisc.bundle.s.Test -Container.com.yahoo.jdisc.bundle.t.Test -Container.com.yahoo.jdisc.bundle.u.Test -Container.com.yahoo.jdisc.client.Test -Container.com.yahoo.jdisc.cloud.aws.Test -Container.com.yahoo.jdisc.core.Test -Container.com.yahoo.jdisc.handler.Test -Container.com.yahoo.jdisc.http.Test -Container.com.yahoo.jdisc.http.cloud.Test -Container.com.yahoo.jdisc.http.filter.Test -Container.com.yahoo.jdisc.http.filter.chain.Test -Container.com.yahoo.jdisc.http.filter.security.athenz.Test -Container.com.yahoo.jdisc.http.filter.security.base.Test -Container.com.yahoo.jdisc.http.filter.security.cors.Test -Container.com.yahoo.jdisc.http.filter.security.csp.Test -Container.com.yahoo.jdisc.http.filter.security.misc.Test -Container.com.yahoo.jdisc.http.filter.security.rule.Test -Container.com.yahoo.jdisc.http.filter.util.Test -Container.com.yahoo.jdisc.http.server.jetty.Test -Container.com.yahoo.jdisc.http.server.jetty.testutils.Test -Container.com.yahoo.jdisc.http.ssl.impl.Test -Container.com.yahoo.jdisc.metrics.yamasconsumer.cloud.Test -Container.com.yahoo.jdisc.refcount.Test -Container.com.yahoo.jdisc.service.Test -Container.com.yahoo.jdisc.statistics.Test -Container.com.yahoo.jdisc.test.Test -Container.com.yahoo.jrt.Test -Container.com.yahoo.jrt.slobrok.Test -Container.com.yahoo.jrt.slobrok.api.Test -Container.com.yahoo.jrt.slobrok.server.Test -Container.com.yahoo.jrt.tool.Test -Container.com.yahoo.lang.Test -Container.com.yahoo.language.Test -Container.com.yahoo.language.detect.Test -Container.com.yahoo.language.opennlp.Test -Container.com.yahoo.language.process.Test -Container.com.yahoo.language.provider.Test -Container.com.yahoo.language.sentencepiece.Test -Container.com.yahoo.language.simple.Test -Container.com.yahoo.language.simple.kstem.Test -Container.com.yahoo.language.tools.Test -Container.com.yahoo.language.wordpiece.Test -Container.com.yahoo.log.Test -Container.com.yahoo.log.event.Test -Container.com.yahoo.log.impl.Test -Container.com.yahoo.logserver.Test -Container.com.yahoo.logserver.filter.Test -Container.com.yahoo.logserver.handlers.Test -Container.com.yahoo.logserver.handlers.archive.Test -Container.com.yahoo.logserver.handlers.logmetrics.Test -Container.com.yahoo.logserver.testutils.Test -Container.com.yahoo.messagebus.Test -Container.com.yahoo.messagebus.jdisc.Test -Container.com.yahoo.messagebus.jdisc.test.Test -Container.com.yahoo.messagebus.network.Test -Container.com.yahoo.messagebus.network.local.Test -Container.com.yahoo.messagebus.network.rpc.Test -Container.com.yahoo.messagebus.network.rpc.test.Test -Container.com.yahoo.messagebus.routing.Test -Container.com.yahoo.messagebus.routing.test.Test -Container.com.yahoo.messagebus.shared.Test -Container.com.yahoo.messagebus.test.Test -Container.com.yahoo.metrics.Test -Container.com.yahoo.metrics.simple.Test -Container.com.yahoo.metrics.simple.jdisc.Test -Container.com.yahoo.metrics.simple.runtime.Test -Container.com.yahoo.nativec.Test -Container.com.yahoo.net.Test -Container.com.yahoo.osgi.Test -Container.com.yahoo.osgi.annotation.Test -Container.com.yahoo.osgi.provider.model.Test -Container.com.yahoo.path.Test -Container.com.yahoo.plugin.Test -Container.com.yahoo.prelude.Test -Container.com.yahoo.prelude.cluster.Test -Container.com.yahoo.prelude.fastsearch.Test -Container.com.yahoo.prelude.hitfield.Test -Container.com.yahoo.prelude.query.Test -Container.com.yahoo.prelude.query.parser.Test -Container.com.yahoo.prelude.query.textualrepresentation.Test -Container.com.yahoo.prelude.querytransform.Test -Container.com.yahoo.prelude.searcher.Test -Container.com.yahoo.prelude.semantics.Test -Container.com.yahoo.prelude.semantics.benchmark.Test -Container.com.yahoo.prelude.semantics.config.Test -Container.com.yahoo.prelude.semantics.engine.Test -Container.com.yahoo.prelude.semantics.parser.Test -Container.com.yahoo.prelude.semantics.rule.Test -Container.com.yahoo.prelude.statistics.Test -Container.com.yahoo.processing.Test -Container.com.yahoo.processing.execution.Test -Container.com.yahoo.processing.execution.chain.Test -Container.com.yahoo.processing.handler.Test -Container.com.yahoo.processing.impl.Test -Container.com.yahoo.processing.processors.Test -Container.com.yahoo.processing.rendering.Test -Container.com.yahoo.processing.request.Test -Container.com.yahoo.processing.request.properties.Test -Container.com.yahoo.processing.response.Test -Container.com.yahoo.processing.test.Test -Container.com.yahoo.protect.Test -Container.com.yahoo.reflection.Test -Container.com.yahoo.restapi.Test -Container.com.yahoo.schema.Test -Container.com.yahoo.schema.derived.Test -Container.com.yahoo.schema.derived.validation.Test -Container.com.yahoo.schema.document.Test -Container.com.yahoo.schema.document.annotation.Test -Container.com.yahoo.schema.expressiontransforms.Test -Container.com.yahoo.schema.fieldoperation.Test -Container.com.yahoo.schema.parser.Test -Container.com.yahoo.schema.processing.Test -Container.com.yahoo.schema.processing.multifieldresolver.Test -Container.com.yahoo.search.Test -Container.com.yahoo.search.cluster.Test -Container.com.yahoo.search.config.Test -Container.com.yahoo.search.dispatch.Test -Container.com.yahoo.search.dispatch.rpc.Test -Container.com.yahoo.search.dispatch.searchcluster.Test -Container.com.yahoo.search.federation.Test -Container.com.yahoo.search.federation.selection.Test -Container.com.yahoo.search.federation.sourceref.Test -Container.com.yahoo.search.grouping.Test -Container.com.yahoo.search.grouping.request.Test -Container.com.yahoo.search.grouping.request.parser.Test -Container.com.yahoo.search.grouping.result.Test -Container.com.yahoo.search.grouping.vespa.Test -Container.com.yahoo.search.handler.Test -Container.com.yahoo.search.handler.observability.Test -Container.com.yahoo.search.intent.model.Test -Container.com.yahoo.searchlib.Test -Container.com.yahoo.searchlib.aggregation.Test -Container.com.yahoo.searchlib.aggregation.hll.Test -Container.com.yahoo.searchlib.document.Test -Container.com.yahoo.searchlib.expression.Test -Container.com.yahoo.searchlib.gbdt.Test -Container.com.yahoo.searchlib.rankingexpression.Test -Container.com.yahoo.searchlib.rankingexpression.evaluation.Test -Container.com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization.Test -Container.com.yahoo.searchlib.rankingexpression.evaluation.tensoroptimization.Test -Container.com.yahoo.searchlib.rankingexpression.parser.Test -Container.com.yahoo.searchlib.rankingexpression.rule.Test -Container.com.yahoo.searchlib.rankingexpression.transform.Test -Container.com.yahoo.searchlib.ranking.features.Test -Container.com.yahoo.searchlib.ranking.features.fieldmatch.Test -Container.com.yahoo.searchlib.tensor.Test -Container.com.yahoo.searchlib.treenet.Test -Container.com.yahoo.searchlib.treenet.parser.Test -Container.com.yahoo.searchlib.treenet.rule.Test -Container.com.yahoo.search.match.Test -Container.com.yahoo.search.pagetemplates.Test -Container.com.yahoo.search.pagetemplates.config.Test -Container.com.yahoo.search.pagetemplates.engine.Test -Container.com.yahoo.search.pagetemplates.engine.resolvers.Test -Container.com.yahoo.search.pagetemplates.model.Test -Container.com.yahoo.search.pagetemplates.result.Test -Container.com.yahoo.search.predicate.Test -Container.com.yahoo.search.predicate.annotator.Test -Container.com.yahoo.search.predicate.benchmarks.Test -Container.com.yahoo.search.predicate.index.Test -Container.com.yahoo.search.predicate.index.conjunction.Test -Container.com.yahoo.search.predicate.optimization.Test -Container.com.yahoo.search.predicate.serialization.Test -Container.com.yahoo.search.predicate.utils.Test -Container.com.yahoo.search.query.Test -Container.com.yahoo.search.query.context.Test -Container.com.yahoo.search.query.gui.Test -Container.com.yahoo.search.query.parser.Test -Container.com.yahoo.search.query.profile.Test -Container.com.yahoo.search.query.profile.compiled.Test -Container.com.yahoo.search.query.profile.config.Test -Container.com.yahoo.search.query.profile.types.Test -Container.com.yahoo.search.query.properties.Test -Container.com.yahoo.search.query.ranking.Test -Container.com.yahoo.search.query.restapi.Test -Container.com.yahoo.search.query.rewrite.Test -Container.com.yahoo.search.query.rewrite.rewriters.Test -Container.com.yahoo.search.query.textserialize.Test -Container.com.yahoo.search.query.textserialize.item.Test -Container.com.yahoo.search.query.textserialize.parser.Test -Container.com.yahoo.search.query.textserialize.serializer.Test -Container.com.yahoo.search.querytransform.Test -Container.com.yahoo.search.rendering.Test -Container.com.yahoo.search.result.Test -Container.com.yahoo.search.schema.Test -Container.com.yahoo.search.schema.internal.Test -Container.com.yahoo.search.searchchain.Test -Container.com.yahoo.search.searchchain.example.Test -Container.com.yahoo.search.searchchain.model.Test -Container.com.yahoo.search.searchchain.model.federation.Test -Container.com.yahoo.search.searchchain.testutil.Test -Container.com.yahoo.search.searchers.Test -Container.com.yahoo.search.statistics.Test -Container.com.yahoo.search.yql.Test -Container.com.yahoo.security.Test -Container.com.yahoo.security.tls.Test -Container.com.yahoo.slime.Test -Container.com.yahoo.socket.test.Test -Container.com.yahoo.stream.Test -Container.com.yahoo.system.Test -Container.com.yahoo.system.execution.Test -Container.com.yahoo.tensor.Test -Container.com.yahoo.tensor.evaluation.Test -Container.com.yahoo.tensor.functions.Test -Container.com.yahoo.tensor.serialization.Test -Container.com.yahoo.test.Test -Container.com.yahoo.test.json.Test -Container.com.yahoo.text.Test -Container.com.yahoo.text.internal.Test -Container.com.yahoo.text.interpretation.Test -Container.com.yahoo.time.Test -Container.com.yahoo.transaction.Test -Container.com.yahoo.vdslib.Test -Container.com.yahoo.vdslib.distribution.Test -Container.com.yahoo.vdslib.state.Test -Container.com.yahoo.vespa.Test -Container.com.yahoo.vespa.applicationmodel.Test -Container.com.yahoo.vespa.athenz.api.Test -Container.com.yahoo.vespa.athenz.aws.Test -Container.com.yahoo.vespa.athenz.client.Test -Container.com.yahoo.vespa.athenz.client.common.Test -Container.com.yahoo.vespa.athenz.client.common.bindings.Test -Container.com.yahoo.vespa.athenz.client.common.serializers.Test -Container.com.yahoo.vespa.athenz.client.zms.Test -Container.com.yahoo.vespa.athenz.client.zms.bindings.Test -Container.com.yahoo.vespa.athenz.client.zts.Test -Container.com.yahoo.vespa.athenz.client.zts.bindings.Test -Container.com.yahoo.vespa.athenz.client.zts.utils.Test -Container.com.yahoo.vespa.athenz.identity.Test -Container.com.yahoo.vespa.athenz.identityprovider.api.Test -Container.com.yahoo.vespa.athenz.identityprovider.api.bindings.Test -Container.com.yahoo.vespa.athenz.identityprovider.client.Test -Container.com.yahoo.vespa.athenz.tls.Test -Container.com.yahoo.vespa.athenz.utils.Test -Container.com.yahoo.vespa.athenz.zpe.Test -Container.com.yahoo.vespaclient.Test -Container.com.yahoo.vespa.clustercontroller.apps.clustercontroller.Test -Container.com.yahoo.vespa.clustercontroller.apputil.communication.http.Test -Container.com.yahoo.vespa.clustercontroller.core.Test -Container.com.yahoo.vespa.clustercontroller.core.database.Test -Container.com.yahoo.vespa.clustercontroller.core.hostinfo.Test -Container.com.yahoo.vespa.clustercontroller.core.listeners.Test -Container.com.yahoo.vespa.clustercontroller.core.restapiv2.Test -Container.com.yahoo.vespa.clustercontroller.core.restapiv2.requests.Test -Container.com.yahoo.vespa.clustercontroller.core.rpc.Test -Container.com.yahoo.vespa.clustercontroller.core.status.Test -Container.com.yahoo.vespa.clustercontroller.core.status.statuspage.Test -Container.com.yahoo.vespa.clustercontroller.utils.communication.async.Test -Container.com.yahoo.vespa.clustercontroller.utils.communication.http.Test -Container.com.yahoo.vespa.clustercontroller.utils.communication.http.writer.Test -Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.Test -Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.Test -Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.Test -Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.response.Test -Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.server.Test -Container.com.yahoo.vespa.clustercontroller.utils.util.Test -Container.com.yahoo.vespa.config.Test -Container.com.yahoo.vespa.config.benchmark.Test -Container.com.yahoo.vespa.config.buildergen.Test -Container.com.yahoo.vespa.config.content.Test -Container.com.yahoo.vespa.config.content.core.Test -Container.com.yahoo.vespa.config.content.reindexing.Test -Container.com.yahoo.vespa.config.core.Test -Container.com.yahoo.vespa.configdefinition.Test -Container.com.yahoo.vespa.configmodel.producers.Test -Container.com.yahoo.vespa.config.parser.Test -Container.com.yahoo.vespa.config.protocol.Test -Container.com.yahoo.vespa.config.proxy.Test -Container.com.yahoo.vespa.config.proxy.filedistribution.Test -Container.com.yahoo.vespa.config.search.Test -Container.com.yahoo.vespa.config.search.core.Test -Container.com.yahoo.vespa.config.server.Test -Container.com.yahoo.vespa.config.server.application.Test -Container.com.yahoo.vespa.config.server.configchange.Test -Container.com.yahoo.vespa.config.server.deploy.Test -Container.com.yahoo.vespa.config.server.filedistribution.Test -Container.com.yahoo.vespa.configserver.flags.Test -Container.com.yahoo.vespa.configserver.flags.db.Test -Container.com.yahoo.vespa.configserver.flags.http.Test -Container.com.yahoo.vespa.config.server.host.Test -Container.com.yahoo.vespa.config.server.http.Test -Container.com.yahoo.vespa.config.server.http.status.Test -Container.com.yahoo.vespa.config.server.http.v1.Test -Container.com.yahoo.vespa.config.server.http.v2.Test -Container.com.yahoo.vespa.config.server.http.v2.request.Test -Container.com.yahoo.vespa.config.server.http.v2.response.Test -Container.com.yahoo.vespa.config.server.maintenance.Test -Container.com.yahoo.vespa.config.server.metrics.Test -Container.com.yahoo.vespa.config.server.model.Test -Container.com.yahoo.vespa.config.server.modelfactory.Test -Container.com.yahoo.vespa.config.server.monitoring.Test -Container.com.yahoo.vespa.config.server.provision.Test -Container.com.yahoo.vespa.config.server.rpc.Test -Container.com.yahoo.vespa.config.server.rpc.security.Test -Container.com.yahoo.vespa.config.server.session.Test -Container.com.yahoo.vespa.config.server.tenant.Test -Container.com.yahoo.vespa.config.server.version.Test -Container.com.yahoo.vespa.config.server.zookeeper.Test -Container.com.yahoo.vespa.config.storage.Test -Container.com.yahoo.vespa.config.util.Test -Container.com.yahoo.vespa.curator.Test -Container.com.yahoo.vespa.curator.api.Test -Container.com.yahoo.vespa.curator.mock.Test -Container.com.yahoo.vespa.curator.recipes.Test -Container.com.yahoo.vespa.curator.stats.Test -Container.com.yahoo.vespa.curator.transaction.Test -Container.com.yahoo.vespa.defaults.Test -Container.com.yahoo.vespa.document.Test -Container.com.yahoo.vespa.document.dom.Test -Container.com.yahoo.vespa.documentmodel.Test -Container.com.yahoo.vespafeeder.Test -Container.com.yahoo.vespa.feed.perf.Test -Container.com.yahoo.vespa.filedistribution.Test -Container.com.yahoo.vespa.filedistribution.status.Test -Container.com.yahoo.vespa.flags.Test -Container.com.yahoo.vespa.flags.custom.Test -Container.com.yahoo.vespa.flags.file.Test -Container.com.yahoo.vespa.flags.json.Test -Container.com.yahoo.vespa.flags.json.wire.Test -Container.com.yahoo.vespaget.Test -Container.com.yahoo.vespa.hadoop.Test -Container.com.yahoo.vespa.hadoop.mapreduce.Test -Container.com.yahoo.vespa.hadoop.mapreduce.util.Test -Container.com.yahoo.vespa.hadoop.pig.Test -Container.com.yahoo.vespa.hosted.athenz.instanceproviderservice.Test -Container.com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.Test -Container.com.yahoo.vespa.hosted.ca.Test -Container.com.yahoo.vespa.hosted.ca.instance.Test -Container.com.yahoo.vespa.hosted.ca.restapi.Test -Container.com.yahoo.vespa.hosted.controller.Test -Container.com.yahoo.vespa.hosted.controller.api.application.v4.Test -Container.com.yahoo.vespa.hosted.controller.api.application.v4.model.Test -Container.com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.Test -Container.com.yahoo.vespa.hosted.controller.api.configserver.Test -Container.com.yahoo.vespa.hosted.controller.api.identifiers.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.archive.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.artifact.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.athenz.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.aws.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.billing.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.certificates.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.configserver.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.deployment.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.dns.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.entity.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.horizon.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.jira.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.maven.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.noderepository.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.organization.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.repair.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.resource.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.routing.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.secrets.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.stubs.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.user.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.vcmr.Test -Container.com.yahoo.vespa.hosted.controller.api.integration.zone.Test -Container.com.yahoo.vespa.hosted.controller.api.role.Test -Container.com.yahoo.vespa.hosted.controller.api.systemflags.v1.Test -Container.com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.Test -Container.com.yahoo.vespa.hosted.controller.application.Test -Container.com.yahoo.vespa.hosted.controller.application.pkg.Test -Container.com.yahoo.vespa.hosted.controller.archive.Test -Container.com.yahoo.vespa.hosted.controller.athenz.Test -Container.com.yahoo.vespa.hosted.controller.athenz.config.Test -Container.com.yahoo.vespa.hosted.controller.athenz.impl.Test -Container.com.yahoo.vespa.hosted.controller.auditlog.Test -Container.com.yahoo.vespa.hosted.controller.certificate.Test -Container.com.yahoo.vespa.hosted.controller.concurrent.Test -Container.com.yahoo.vespa.hosted.controller.deployment.Test -Container.com.yahoo.vespa.hosted.controller.dns.Test -Container.com.yahoo.vespa.hosted.controller.maintenance.Test -Container.com.yahoo.vespa.hosted.controller.metric.Test -Container.com.yahoo.vespa.hosted.controller.notification.Test -Container.com.yahoo.vespa.hosted.controller.persistence.Test -Container.com.yahoo.vespa.hosted.controller.proxy.Test -Container.com.yahoo.vespa.hosted.controller.restapi.application.Test -Container.com.yahoo.vespa.hosted.controller.restapi.athenz.Test -Container.com.yahoo.vespa.hosted.controller.restapi.billing.Test -Container.com.yahoo.vespa.hosted.controller.restapi.changemanagement.Test -Container.com.yahoo.vespa.hosted.controller.restapi.configserver.Test -Container.com.yahoo.vespa.hosted.controller.restapi.controller.Test -Container.com.yahoo.vespa.hosted.controller.restapi.deployment.Test -Container.com.yahoo.vespa.hosted.controller.restapi.filter.Test -Container.com.yahoo.vespa.hosted.controller.restapi.flags.Test -Container.com.yahoo.vespa.hosted.controller.restapi.horizon.Test -Container.com.yahoo.vespa.hosted.controller.restapi.os.Test -Container.com.yahoo.vespa.hosted.controller.restapi.routing.Test -Container.com.yahoo.vespa.hosted.controller.restapi.systemflags.Test -Container.com.yahoo.vespa.hosted.controller.restapi.user.Test -Container.com.yahoo.vespa.hosted.controller.restapi.zone.v1.Test -Container.com.yahoo.vespa.hosted.controller.restapi.zone.v2.Test -Container.com.yahoo.vespa.hosted.controller.routing.Test -Container.com.yahoo.vespa.hosted.controller.routing.context.Test -Container.com.yahoo.vespa.hosted.controller.routing.rotation.Test -Container.com.yahoo.vespa.hosted.controller.security.Test -Container.com.yahoo.vespa.hosted.controller.support.access.Test -Container.com.yahoo.vespa.hosted.controller.tenant.Test -Container.com.yahoo.vespa.hosted.controller.tls.Test -Container.com.yahoo.vespa.hosted.controller.versions.Test -Container.com.yahoo.vespa.hosted.node.admin.Test -Container.com.yahoo.vespa.hosted.node.admin.component.Test -Container.com.yahoo.vespa.hosted.node.admin.configserver.Test -Container.com.yahoo.vespa.hosted.node.admin.configserver.flags.Test -Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Test -Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.Test -Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.Test -Container.com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Test -Container.com.yahoo.vespa.hosted.node.admin.configserver.state.Test -Container.com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.Test -Container.com.yahoo.vespa.hosted.node.admin.container.Test -Container.com.yahoo.vespa.hosted.node.admin.container.image.Test -Container.com.yahoo.vespa.hosted.node.admin.container.metrics.Test -Container.com.yahoo.vespa.hosted.node.admin.maintenance.Test -Container.com.yahoo.vespa.hosted.node.admin.maintenance.acl.Test -Container.com.yahoo.vespa.hosted.node.admin.maintenance.coredump.Test -Container.com.yahoo.vespa.hosted.node.admin.maintenance.disk.Test -Container.com.yahoo.vespa.hosted.node.admin.maintenance.identity.Test -Container.com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Test -Container.com.yahoo.vespa.hosted.node.admin.maintenance.sync.Test -Container.com.yahoo.vespa.hosted.node.admin.nodeadmin.Test -Container.com.yahoo.vespa.hosted.node.admin.nodeagent.Test -Container.com.yahoo.vespa.hosted.node.admin.provider.Test -Container.com.yahoo.vespa.hosted.node.admin.task.util.Test -Container.com.yahoo.vespa.hosted.node.admin.task.util.editor.Test -Container.com.yahoo.vespa.hosted.node.admin.task.util.file.Test -Container.com.yahoo.vespa.hosted.node.admin.task.util.fs.Test -Container.com.yahoo.vespa.hosted.node.admin.task.util.network.Test -Container.com.yahoo.vespa.hosted.node.admin.task.util.process.Test -Container.com.yahoo.vespa.hosted.node.admin.task.util.systemd.Test -Container.com.yahoo.vespa.hosted.node.admin.task.util.template.Test -Container.com.yahoo.vespa.hosted.node.admin.task.util.text.Test -Container.com.yahoo.vespa.hosted.node.admin.task.util.yum.Test -Container.com.yahoo.vespa.hosted.provision.Test -Container.com.yahoo.vespa.hosted.provision.applications.Test -Container.com.yahoo.vespa.hosted.provision.autoscale.Test -Container.com.yahoo.vespa.hosted.provision.lb.Test -Container.com.yahoo.vespa.hosted.provision.maintenance.Test -Container.com.yahoo.vespa.hosted.provision.node.Test -Container.com.yahoo.vespa.hosted.provision.node.filter.Test -Container.com.yahoo.vespa.hosted.provision.os.Test -Container.com.yahoo.vespa.hosted.provision.persistence.Test -Container.com.yahoo.vespa.hosted.provision.provisioning.Test -Container.com.yahoo.vespa.hosted.provision.restapi.Test -Container.com.yahoo.vespa.hosted.provision.testutils.Test -Container.com.yahoo.vespa.hosted.routing.Test -Container.com.yahoo.vespa.hosted.routing.nginx.Test -Container.com.yahoo.vespa.hosted.routing.restapi.Test -Container.com.yahoo.vespa.hosted.routing.status.Test -Container.com.yahoo.vespa.hosted.testrunner.Test -Container.com.yahoo.vespa.http.server.Test -Container.com.yahoo.vespa.http.server.util.Test -Container.com.yahoo.vespa.indexinglanguage.Test -Container.com.yahoo.vespa.indexinglanguage.expressions.Test -Container.com.yahoo.vespa.indexinglanguage.linguistics.Test -Container.com.yahoo.vespa.indexinglanguage.parser.Test -Container.com.yahoo.vespa.indexinglanguage.predicate.Test -Container.com.yahoo.vespa.jaxrs.annotation.Test -Container.com.yahoo.vespa.maven.plugin.enforcer.Test -Container.com.yahoo.vespa.model.Test -Container.com.yahoo.vespa.model.admin.Test -Container.com.yahoo.vespa.model.admin.clustercontroller.Test -Container.com.yahoo.vespa.model.admin.metricsproxy.Test -Container.com.yahoo.vespa.model.admin.monitoring.Test -Container.com.yahoo.vespa.model.admin.monitoring.builder.Test -Container.com.yahoo.vespa.model.admin.monitoring.builder.xml.Test -Container.com.yahoo.vespa.model.application.validation.Test -Container.com.yahoo.vespa.model.application.validation.change.Test -Container.com.yahoo.vespa.model.application.validation.change.search.Test -Container.com.yahoo.vespa.model.application.validation.first.Test -Container.com.yahoo.vespa.model.builder.Test -Container.com.yahoo.vespa.model.builder.xml.dom.Test -Container.com.yahoo.vespa.model.builder.xml.dom.chains.Test -Container.com.yahoo.vespa.model.builder.xml.dom.chains.docproc.Test -Container.com.yahoo.vespa.model.builder.xml.dom.chains.processing.Test -Container.com.yahoo.vespa.model.builder.xml.dom.chains.search.Test -Container.com.yahoo.vespa.model.clients.Test -Container.com.yahoo.vespa.model.container.Test -Container.com.yahoo.vespa.model.container.component.Test -Container.com.yahoo.vespa.model.container.component.chain.Test -Container.com.yahoo.vespa.model.container.configserver.Test -Container.com.yahoo.vespa.model.container.configserver.option.Test -Container.com.yahoo.vespa.model.container.docproc.Test -Container.com.yahoo.vespa.model.container.docproc.model.Test -Container.com.yahoo.vespa.model.container.http.Test -Container.com.yahoo.vespa.model.container.http.ssl.Test -Container.com.yahoo.vespa.model.container.http.xml.Test -Container.com.yahoo.vespa.model.container.ml.Test -Container.com.yahoo.vespa.model.container.processing.Test -Container.com.yahoo.vespa.model.container.search.Test -Container.com.yahoo.vespa.model.container.search.searchchain.Test -Container.com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains.Test -Container.com.yahoo.vespa.model.container.xml.Test -Container.com.yahoo.vespa.model.container.xml.document.Test -Container.com.yahoo.vespa.model.container.xml.embedder.Test -Container.com.yahoo.vespa.model.content.Test -Container.com.yahoo.vespa.model.content.cluster.Test -Container.com.yahoo.vespa.model.content.engines.Test -Container.com.yahoo.vespa.model.content.storagecluster.Test -Container.com.yahoo.vespa.model.filedistribution.Test -Container.com.yahoo.vespa.model.ml.Test -Container.com.yahoo.vespa.model.routing.Test -Container.com.yahoo.vespa.model.search.Test -Container.com.yahoo.vespa.model.utils.Test -Container.com.yahoo.vespa.model.utils.internal.Test -Container.com.yahoo.vespa.objects.Test -Container.com.yahoo.vespa.orchestrator.Test -Container.com.yahoo.vespa.orchestrator.config.Test -Container.com.yahoo.vespa.orchestrator.controller.Test -Container.com.yahoo.vespa.orchestrator.model.Test -Container.com.yahoo.vespa.orchestrator.policy.Test -Container.com.yahoo.vespa.orchestrator.resources.Test -Container.com.yahoo.vespa.orchestrator.restapi.wire.Test -Container.com.yahoo.vespa.orchestrator.status.Test -Container.com.yahoo.vespa.orchestrator.status.json.Test -Container.com.yahoo.vespa.security.tool.securityenv.Test -Container.com.yahoo.vespa.service.duper.Test -Container.com.yahoo.vespa.service.executor.Test -Container.com.yahoo.vespa.service.health.Test -Container.com.yahoo.vespa.service.manager.Test -Container.com.yahoo.vespa.service.model.Test -Container.com.yahoo.vespa.service.monitor.Test -Container.com.yahoo.vespa.service.slobrok.Test -Container.com.yahoo.vespastat.Test -Container.com.yahoo.vespa.streamingvisitors.Test -Container.com.yahoo.vespa.streamingvisitors.tracing.Test -Container.com.yahoo.vespasummarybenchmark.Test -Container.com.yahoo.vespa.test.file.Test -Container.com.yahoo.vespa.testrunner.Test -Container.com.yahoo.vespavisit.Test -Container.com.yahoo.vespaxmlparser.Test -Container.com.yahoo.vespa.zookeeper.Test -Container.com.yahoo.vespa.zookeeper.cli.Test -Container.com.yahoo.vespa.zookeeper.client.Test -Container.com.yahoo.yolean.Test -Container.com.yahoo.yolean.chain.Test -Container.com.yahoo.yolean.concurrent.Test -Container.com.yahoo.yolean.function.Test -Container.com.yahoo.yolean.system.Test -Container.com.yahoo.yolean.trace.Test diff --git a/client/go/internal/admin/script-utils/logfmt/internal_notnames.txt b/client/go/internal/admin/script-utils/logfmt/internal_notnames.txt deleted file mode 100644 index 49543758ac3..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/internal_notnames.txt +++ /dev/null @@ -1,8 +0,0 @@ -Container.ai.onnx.Test -Container.com.fasterxml.jackson.jaxrs.json.Test -Container.com.google.Test -Container.com.yahooapis.foo.Test -Container.com.yahoo.newssearch.Test -Container.com.yahoo.Test -Container.com.yahoo.testing.Test -Container.org.apache.http.Test diff --git a/client/go/internal/admin/script-utils/logfmt/internal_test.go b/client/go/internal/admin/script-utils/logfmt/internal_test.go deleted file mode 100644 index 9b6b0f8404c..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/internal_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package logfmt - -import ( - "bufio" - "os" - "testing" -) - -// tests: func isInternal(componentName string) bool - -func TestIsInternal(t *testing.T) { - f, err := os.Open("internal_names.txt") - if err != nil { - t.Fatal("could not read test data") - } - defer f.Close() - for input := bufio.NewScanner(f); input.Scan(); { - if name := input.Text(); !isInternal(name) { - t.Logf("name '%s' should be internal but was not recognized", name) - t.Fail() - } - } - f, err = os.Open("internal_notnames.txt") - if err != nil { - t.Fatal("could not read test data") - } - defer f.Close() - for input := bufio.NewScanner(f); input.Scan(); { - if name := input.Text(); isInternal(name) { - t.Logf("name '%s' should not be internal but was recognized", name) - t.Fail() - } - } -} diff --git a/client/go/internal/admin/script-utils/logfmt/levelflags.go b/client/go/internal/admin/script-utils/logfmt/levelflags.go deleted file mode 100644 index 4e6c1284753..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/levelflags.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -package logfmt - -import ( - "strings" -) - -// handle CLI flags for log level filtering - -type flagValueForLevel struct { - levels map[string]bool - changed bool -} - -func defaultLevelFlags() map[string]bool { - return map[string]bool{ - "fatal": true, - "error": true, - "warning": true, - "info": true, - "config": false, - "event": false, - "debug": false, - "spam": false, - } -} - -func (v *flagValueForLevel) Type() string { - return "level flags" -} - -func (v *flagValueForLevel) String() string { - var buf strings.Builder - flagNames := []string{ - "fatal", - "error", - "warning", - "info", - "config", - "event", - "debug", - "spam", - } - for _, flag := range flagNames { - if v.levels[flag] { - buf.WriteString(" +") - } else { - buf.WriteString(" -") - } - buf.WriteString(flag) - } - return buf.String() -} - -func (v *flagValueForLevel) flags() map[string]bool { - return v.levels -} - -func (v *flagValueForLevel) name() string { - return "level" -} - -func (v *flagValueForLevel) unchanged() bool { - return !v.changed -} - -func (v *flagValueForLevel) Set(val string) error { - return applyPlusMinus(val, v) -} diff --git a/client/go/internal/admin/script-utils/logfmt/levelflags_test.go b/client/go/internal/admin/script-utils/logfmt/levelflags_test.go deleted file mode 100644 index 186ea2d96b0..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/levelflags_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -package logfmt - -import ( - "strings" - "testing" -) - -func TestLevelFlags(t *testing.T) { - none := " -fatal -error -warning -info -config -event -debug -spam" - - var flag flagValueForLevel - if flag.String() != none { - t.Logf("unset flag displays as '%s', expected '%s'", flag.String(), none) - t.Fail() - } - if flag.Type() != "level flags" { - t.Logf("flag type was '%s'", flag.Type()) - t.Fail() - } - check := func(expected string, texts ...string) { - var target flagValueForLevel - // target.levels = defaultLevelFlags() - target.levels = defaultLevelFlags() - for _, text := range texts { - err := target.Set(text) - if err != nil { - t.Fatalf("unexpected error with level flags Set('%s'): %v", text, err) - } - } - got := target.String() - if got != expected { - t.Logf("expected flags [%s] but got: [%s]", expected, got) - t.Fail() - } - } - check(" +fatal +error +warning +info -config -event -debug -spam") - check(" -fatal -error -warning -info -config -event -debug -spam", "-all") - check(" +fatal +error +warning +info +config +event +debug +spam", "all") - check(" +fatal +error +warning +info +config +event +debug +spam", "+all") - check(" -fatal -error -warning -info -config -event +debug -spam", "debug") - check(" +fatal +error +warning +info +config +event +debug -spam", "all-spam") - check(" +fatal +error +warning +info +config +event +debug -spam", "all", "-spam") - check(" +fatal +error +warning -info -config +event -debug -spam", "+event", "-info") - check(" +fatal +error -warning -info -config +event -debug -spam", "+event,-info,-warning,config") - check(" +fatal +error -warning -info +config +event -debug -spam", "+event,-info,-warning,+config") - check(" +fatal +error -warning -info +config +event -debug -spam", "+event,-info", "-warning,+config") - check(" -fatal -error -warning -info +config -event -debug -spam", "+event", "-info", "-warning", "config") - check = func(expectErr string, texts ...string) { - var target flagValueForLevel - target.levels = defaultLevelFlags() - for _, text := range texts { - err := target.Set(text) - if err != nil { - if err.Error() == expectErr { - return - } - t.Fatalf("expected error [%s] with level flags Set('%s'), but got [%v]", expectErr, text, err) - } - } - t.Logf("Did not get expected error '%s' from %s", expectErr, strings.Join(texts, ",")) - t.Fail() - } - check("not a valid level flag: 'foo'", "foo") - check("not a valid level flag: 'foo'", "event,foo,config") - check("not a valid level flag: 'foo'", "-event,-foo,-config") - check("not a valid level flag: 'foo'", "+event,+foo,+config") - check("not a valid level flag: 'foo'", "event", "foo", "config") - check("not a valid level flag: 'foo'", "-event", "-foo", "-config") - check("not a valid level flag: 'foo'", "+event", "+foo", "+config") -} diff --git a/client/go/internal/admin/script-utils/logfmt/options.go b/client/go/internal/admin/script-utils/logfmt/options.go deleted file mode 100644 index 864868d4ce5..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/options.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -package logfmt - -import ( - "fmt" - "os" -) - -// options designed for compatibility with perl version of vespa-logfmt - -type Options struct { - ShowFields flagValueForShow - ShowLevels flagValueForLevel - OnlyHostname string - OnlyPid string - OnlyService string - OnlyInternal bool - FollowTail bool - DequoteNewlines bool - TruncateService bool - TruncateComponent bool - ComponentFilter regexFlag - MessageFilter regexFlag - Format OutputFormat -} - -func NewOptions() (ret Options) { - ret.ShowLevels.levels = defaultLevelFlags() - ret.ShowFields.shown = defaultShowFlags() - return -} - -func (o *Options) showField(field string) bool { - return o.ShowFields.shown[field] -} - -func (o *Options) showLevel(level string) bool { - rv, ok := o.ShowLevels.levels[level] - if !ok { - o.ShowLevels.levels[level] = true - fmt.Fprintf(os.Stderr, "Warnings: unknown level '%s' in input\n", level) - return true - } - return rv -} diff --git a/client/go/internal/admin/script-utils/logfmt/plusminusflag.go b/client/go/internal/admin/script-utils/logfmt/plusminusflag.go deleted file mode 100644 index 1768cf0e7be..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/plusminusflag.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -package logfmt - -import ( - "fmt" - "strings" -) - -// common code for showFlags and levelFlags - -type plusMinusFlag interface { - flags() map[string]bool - name() string - unchanged() bool -} - -func trimPrefix(value, prefix string) (newValue string, hadPrefix bool) { - hadPrefix = strings.HasPrefix(value, prefix) - if hadPrefix { - newValue = strings.TrimPrefix(value, prefix) - } else { - newValue = value - } - return -} - -func applyPlusMinus(val string, target plusMinusFlag) error { - minus := strings.HasPrefix(val, "-") - plus := strings.HasPrefix(val, "+") - val = strings.ReplaceAll(val, "-", ",-") - val = strings.ReplaceAll(val, "+", ",+") - if target.unchanged() { - // user wants to reset flags? - if minus == false && plus == false { - for k, _ := range target.flags() { - target.flags()[k] = false - } - } - } - changeTo := !minus - for _, k := range strings.Split(val, ",") { - if suppress, minus := trimPrefix(k, "-"); minus { - k = suppress - changeTo = false - } - if surface, plus := trimPrefix(k, "+"); plus { - k = surface - changeTo = true - } - if k == "" { - continue - } - if k == "all" { - for k, _ := range target.flags() { - target.flags()[k] = changeTo - } - } else if _, ok := target.flags()[k]; !ok { - return fmt.Errorf("not a valid %s flag: '%s'", target.name(), k) - } else { - target.flags()[k] = changeTo - } - } - return nil -} diff --git a/client/go/internal/admin/script-utils/logfmt/regexflag.go b/client/go/internal/admin/script-utils/logfmt/regexflag.go deleted file mode 100644 index 8f7d2a91373..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/regexflag.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -package logfmt - -import ( - "regexp" -) - -// optional regular expression filter, as a CLI flag - -type regexFlag struct { - regex *regexp.Regexp -} - -func (re regexFlag) unmatched(s string) bool { - if re.regex == nil { - return false - } - return re.regex.FindStringIndex(s) == nil -} - -func (v *regexFlag) Type() string { - return "regular expression" -} - -func (v *regexFlag) String() string { - if v.regex == nil { - return "" - } - return v.regex.String() -} - -func (v *regexFlag) Set(val string) (r error) { - v.regex, r = regexp.Compile(val) - return -} diff --git a/client/go/internal/admin/script-utils/logfmt/regexflag_test.go b/client/go/internal/admin/script-utils/logfmt/regexflag_test.go deleted file mode 100644 index 489439863a2..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/regexflag_test.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -package logfmt - -import ( - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "testing" -) - -func assertMatch(t *testing.T, re string, flag *regexFlag, texts ...string) { - t.Helper() - err := flag.Set(re) - require.Nil(t, err, "unexpected error with flag.Set('%s'): %v", re, err) - assert.Equal(t, re, flag.String(), "set flag displays as '%s', expected '%s'", flag.String(), re) - for _, text := range texts { - assert.False(t, flag.unmatched(text), "flag '%s' claims a non-match for '%s'", flag.String(), text) - } -} - -func assertUnmatch(t *testing.T, re string, flag *regexFlag, texts ...string) { - t.Helper() - err := flag.Set(re) - require.Nil(t, err, "unexpected error with flag.Set('%s'): %v", re, err) - assert.Equal(t, re, flag.String()) - for _, text := range texts { - assert.True(t, flag.unmatched(text), "flag '%s' should claim a non-match for '%s'", flag.String(), text) - } -} - -func TestRegexFlag(t *testing.T) { - var flag regexFlag - assert.Equal(t, "", flag.String()) - assert.Equal(t, "regular expression", flag.Type()) - assert.False(t, flag.unmatched("foobar"), "unset flag claims a non-match") - assert.EqualError(t, flag.Set("*"), "error parsing regexp: missing argument to repetition operator: `*`") - assertMatch(t, "foo.*bar", new(regexFlag), "foobar", "foo bar", "x foobar y", "xfoobary", "xfooybarz") - assertUnmatch(t, "foo.*bar", new(regexFlag), "Foobar", "foo Bar", "fxoobar", "whatever") -} diff --git a/client/go/internal/admin/script-utils/logfmt/runlogfmt.go b/client/go/internal/admin/script-utils/logfmt/runlogfmt.go deleted file mode 100644 index 6557461598e..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/runlogfmt.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -package logfmt - -import ( - "bufio" - "fmt" - "os" - - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func inputIsPipe() bool { - fi, err := os.Stdin.Stat() - if err != nil { - return false - } - if fi.Mode()&os.ModeNamedPipe == 0 { - return false - } else { - return true - } -} - -// main entry point for vespa-logfmt - -func RunLogfmt(opts *Options, args []string) { - if len(args) == 0 { - if !inputIsPipe() { - args = append(args, vespa.FindHome()+"/logs/vespa/vespa.log") - } else { - formatFile(opts, os.Stdin) - } - } - if opts.FollowTail { - if len(args) != 1 { - fmt.Fprintf(os.Stderr, "Must have exact 1 file for 'follow' option, got %d\n", len(args)) - return - } - if err := tailFile(opts, args[0]); err != nil { - fmt.Fprintln(os.Stderr, err) - return - } - return - } - for _, arg := range args { - file, err := os.Open(arg) - if err != nil { - fmt.Fprintf(os.Stderr, "Cannot open '%s': %v\n", arg, err) - } else { - formatFile(opts, file) - file.Close() - } - } -} - -func formatLine(opts *Options, line string) { - output, err := handleLine(opts, line) - if err != nil { - fmt.Fprintln(os.Stderr, "bad log line:", err) - } else { - os.Stdout.WriteString(output) - } -} - -func tailFile(opts *Options, fn string) error { - tailed, err := FollowFile(fn) - if err != nil { - return err - } - for line := range tailed.Lines() { - formatLine(opts, line.Text) - } - return nil -} - -func formatFile(opts *Options, arg *os.File) { - input := bufio.NewScanner(arg) - input.Buffer(make([]byte, 64*1024), 4*1024*1024) - for input.Scan() { - formatLine(opts, input.Text()) - } -} diff --git a/client/go/internal/admin/script-utils/logfmt/showflags.go b/client/go/internal/admin/script-utils/logfmt/showflags.go deleted file mode 100644 index b69860e0312..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/showflags.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -package logfmt - -import ( - "strings" -) - -// handle CLI flags for which fields to show when formatting a line - -type flagValueForShow struct { - shown map[string]bool - changed bool -} - -func defaultShowFlags() map[string]bool { - return map[string]bool{ - "time": true, - "fmttime": true, - "msecs": true, - "usecs": false, - "host": false, - "level": true, - "pid": false, - "service": true, - "component": true, - "message": true, - } -} - -func (v *flagValueForShow) Type() string { - return "show flags" -} - -func (v *flagValueForShow) String() string { - var buf strings.Builder - flagNames := []string{ - "time", - "fmttime", - "msecs", - "usecs", - "host", - "level", - "pid", - "service", - "component", - "message", - } - for _, flag := range flagNames { - if v.shown[flag] { - buf.WriteString(" +") - } else { - buf.WriteString(" -") - } - buf.WriteString(flag) - } - return buf.String() -} - -func (v *flagValueForShow) flags() map[string]bool { - return v.shown -} - -func (v *flagValueForShow) name() string { - return "show" -} - -func (v *flagValueForShow) unchanged() bool { - return !v.changed -} - -func (v *flagValueForShow) Set(val string) error { - return applyPlusMinus(val, v) -} diff --git a/client/go/internal/admin/script-utils/logfmt/showflags_test.go b/client/go/internal/admin/script-utils/logfmt/showflags_test.go deleted file mode 100644 index d1b66118afd..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/showflags_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -package logfmt - -import ( - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "strings" - "testing" -) - -func TestShowFlags(t *testing.T) { - none := " -time -fmttime -msecs -usecs -host -level -pid -service -component -message" - var flag flagValueForShow - assert.Equal(t, none, flag.String(), "unset flag displays as '%s', expected '%s'", flag.String(), none) - assert.Equal(t, "show flags", flag.Type()) - check := func(expected string, texts ...string) { - var target flagValueForShow - // target.levels = defaultLevelFlags() - target.shown = defaultShowFlags() - for _, text := range texts { - err := target.Set(text) - require.Nil(t, err, "unexpected error with show flags Set('%s'): %v", text, err) - } - assert.Equal(t, expected, target.String()) - } - check(" +time +fmttime +msecs -usecs -host +level -pid +service +component +message") - check(" -time -fmttime -msecs -usecs -host -level -pid -service -component -message", "-all") - check(" +time +fmttime +msecs +usecs +host +level +pid +service +component +message", "all") - check(" +time +fmttime +msecs +usecs +host +level +pid +service +component +message", "+all") - check(" -time -fmttime -msecs -usecs -host -level +pid -service -component -message", "pid") - check(" +time +fmttime +msecs +usecs -host +level +pid +service +component +message", "all-host") - check(" +time +fmttime +msecs +usecs -host +level +pid +service +component +message", "all", "-host") - check(" +time +fmttime -msecs -usecs -host +level +pid +service +component +message", "+pid", "-msecs") - check(" +time -fmttime +msecs -usecs +host +level -pid -service +component +message", "+host,-fmttime,-service,pid") - check(" +time -fmttime +msecs -usecs +host +level +pid -service +component +message", "+host,-fmttime,-service,+pid") - check(" +time -fmttime +msecs -usecs +host +level +pid -service +component +message", "+host,-fmttime", "-service,+pid") - check(" -time -fmttime -msecs -usecs -host -level +pid -service -component -message", "+host", "-fmttime", "-service", "pid") - check = func(expectErr string, texts ...string) { - var target flagValueForShow - target.shown = defaultShowFlags() - for _, text := range texts { - err := target.Set(text) - if err != nil { - require.Equal(t, expectErr, err.Error()) - return - } - } - t.Logf("Did not get expected error [%s] from %s", expectErr, strings.Join(texts, " ")) - t.Fail() - } - check("not a valid show flag: 'foo'", "foo") - check("not a valid show flag: 'foo'", "level,foo,message") - check("not a valid show flag: 'foo'", "-level,-foo,-message") - check("not a valid show flag: 'foo'", "+level,+foo,+message") - check("not a valid show flag: 'foo'", "level", "foo", "message") - check("not a valid show flag: 'foo'", "-level", "-foo", "-message") - check("not a valid show flag: 'foo'", "+level", "+foo", "+message") -} diff --git a/client/go/internal/admin/script-utils/logfmt/tail.go b/client/go/internal/admin/script-utils/logfmt/tail.go deleted file mode 100644 index 75e7cbb0693..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/tail.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: mpolden - -package logfmt - -type Line struct { - Text string -} - -type Tail interface { - Lines() chan Line -} diff --git a/client/go/internal/admin/script-utils/logfmt/tail_not_unix.go b/client/go/internal/admin/script-utils/logfmt/tail_not_unix.go deleted file mode 100644 index 7030572575d..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/tail_not_unix.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: mpolden - -//go:build windows - -package logfmt - -import ( - "fmt" -) - -func FollowFile(fn string) (Tail, error) { - return nil, fmt.Errorf("tail is not supported on this platform") -} diff --git a/client/go/internal/admin/script-utils/logfmt/tail_unix.go b/client/go/internal/admin/script-utils/logfmt/tail_unix.go deleted file mode 100644 index 7703844da48..00000000000 --- a/client/go/internal/admin/script-utils/logfmt/tail_unix.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa logfmt command -// Author: arnej - -//go:build !windows - -package logfmt - -import ( - "bufio" - "fmt" - "io" - "os" - "time" - - "golang.org/x/sys/unix" -) - -const lastLinesSize = 4 * 1024 - -// an active "tail -f" like object - -type unixTail struct { - lines chan Line - lineBuf []byte - curFile *os.File - fn string - reader *bufio.Reader - curStat unix.Stat_t -} - -func (t *unixTail) Lines() chan Line { return t.lines } - -// API for starting to follow a log file - -func FollowFile(fn string) (Tail, error) { - res := unixTail{} - res.fn = fn - res.lineBuf = make([]byte, lastLinesSize) - res.openTail() - res.lines = make(chan Line, 20) - res.lineBuf = res.lineBuf[:0] - go runTailWith(&res) - return &res, nil -} - -func (t *unixTail) setFile(f *os.File) { - if t.curFile != nil { - t.curFile.Close() - } - t.curFile = f - if f != nil { - err := unix.Fstat(int(f.Fd()), &t.curStat) - if err != nil { - f.Close() - fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) - return - } - t.reader = bufio.NewReaderSize(f, 1024*1024) - } else { - t.reader = nil - } -} - -// open log file and seek to the start of a line near the end, if possible. -func (t *unixTail) openTail() { - file, err := os.Open(t.fn) - if err != nil { - return - } - sz, err := file.Seek(0, os.SEEK_END) - if err != nil { - return - } - if sz < lastLinesSize { - sz, err = file.Seek(0, os.SEEK_SET) - if err == nil { - // just read from start of file, all OK - t.setFile(file) - } - return - } - sz, _ = file.Seek(-lastLinesSize, os.SEEK_END) - n, err := file.Read(t.lineBuf) - if err != nil { - return - } - for i := 0; i < n; i++ { - if t.lineBuf[i] == '\n' { - sz, err = file.Seek(sz+int64(i+1), os.SEEK_SET) - if err == nil { - t.setFile(file) - } - return - } - } -} - -func (t *unixTail) reopen(cur *unix.Stat_t) { - for cnt := 0; cnt < 100; cnt++ { - file, err := os.Open(t.fn) - if err != nil { - t.setFile(nil) - if cnt == 0 { - fmt.Fprintf(os.Stderr, "%v (waiting for log file to appear)\n", err) - } - time.Sleep(1000 * time.Millisecond) - continue - } - var stat unix.Stat_t - err = unix.Fstat(int(file.Fd()), &stat) - if err != nil { - file.Close() - fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) - time.Sleep(5000 * time.Millisecond) - continue - } - if cur != nil && cur.Dev == stat.Dev && cur.Ino == stat.Ino { - // same file, continue following it - file.Close() - return - } - // new file, start following it - t.setFile(file) - return - } -} - -// runs as a goroutine -func runTailWith(t *unixTail) { - defer t.setFile(nil) -loop: - for { - for t.curFile == nil { - t.reopen(nil) - } - bytes, err := t.reader.ReadSlice('\n') - t.lineBuf = append(t.lineBuf, bytes...) - if err == bufio.ErrBufferFull { - continue - } - if err == nil { - ll := len(t.lineBuf) - 1 - t.lines <- Line{Text: string(t.lineBuf[:ll])} - t.lineBuf = t.lineBuf[:0] - continue - } - if err == io.EOF { - pos, _ := t.curFile.Seek(0, os.SEEK_CUR) - for cnt := 0; cnt < 100; cnt++ { - time.Sleep(10 * time.Millisecond) - sz, _ := t.curFile.Seek(0, os.SEEK_END) - if sz != pos { - if sz < pos { - // truncation case - pos = 0 - } - t.curFile.Seek(pos, os.SEEK_SET) - continue loop - } - } - // no change in file size, try reopening - t.reopen(&t.curStat) - } else { - fmt.Fprintf(os.Stderr, "error tailing '%s': %v\n", t.fn, err) - close(t.lines) - return - } - } -} diff --git a/client/go/internal/admin/script-utils/main.go b/client/go/internal/admin/script-utils/main.go deleted file mode 100644 index 78948908a64..00000000000 --- a/client/go/internal/admin/script-utils/main.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package main - -import ( - "fmt" - "os" - "strings" - - "github.com/vespa-engine/vespa/client/go/internal/admin/clusterstate" - "github.com/vespa-engine/vespa/client/go/internal/admin/deploy" - "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" - "github.com/vespa-engine/vespa/client/go/internal/admin/script-utils/configserver" - "github.com/vespa-engine/vespa/client/go/internal/admin/script-utils/logfmt" - "github.com/vespa-engine/vespa/client/go/internal/admin/script-utils/services" - "github.com/vespa-engine/vespa/client/go/internal/admin/script-utils/standalone" - "github.com/vespa-engine/vespa/client/go/internal/admin/script-utils/startcbinary" - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func basename(s string) string { - parts := strings.Split(s, "/") - return parts[len(parts)-1] -} - -func main() { - defer handleSimplePanic() - _ = vespa.FindAndVerifyVespaHome() - action := basename(os.Args[0]) - if action == "script-utils" && len(os.Args) > 1 { - action = os.Args[1] - os.Args = os.Args[1:] - } - switch action { - case "vespa-stop-services": - os.Exit(services.VespaStopServices()) - case "vespa-start-services": - os.Exit(services.VespaStartServices()) - case "start-services": - os.Exit(services.StartServices()) - case "just-run-configproxy": - os.Exit(services.JustRunConfigproxy()) - case "vespa-start-configserver": - os.Exit(configserver.StartConfigserverEtc()) - case "just-start-configserver": - os.Exit(configserver.JustStartConfigserver()) - case "vespa-start-container-daemon": - os.Exit(jvm.RunApplicationContainer(os.Args[1:])) - case "run-standalone-container": - os.Exit(standalone.StartStandaloneContainer(os.Args[1:])) - case "start-c-binary": - os.Exit(startcbinary.Run(os.Args[1:])) - case "export-env": - vespa.ExportDefaultEnvToSh() - case "security-env", "vespa-security-env": - vespa.ExportSecurityEnvToSh() - case "ipv6-only": - if vespa.HasOnlyIpV6() { - os.Exit(0) - } else { - os.Exit(1) - } - case "detect-hostname": - myName, err := vespa.FindOurHostname() - fmt.Println(myName) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - case "vespa-deploy": - cobra := deploy.NewDeployCmd() - cobra.Execute() - case "vespa-logfmt": - cobra := logfmt.NewLogfmtCmd() - cobra.Execute() - case "vespa-get-cluster-state": - cobra := clusterstate.NewGetClusterStateCmd() - cobra.Execute() - case "vespa-get-node-state": - cobra := clusterstate.NewGetNodeStateCmd() - cobra.Execute() - case "vespa-set-node-state": - cobra := clusterstate.NewSetNodeStateCmd() - cobra.Execute() - default: - if startcbinary.IsCandidate(os.Args[0]) { - os.Exit(startcbinary.Run(os.Args)) - } - fmt.Fprintf(os.Stderr, "unknown action '%s'\n", action) - fmt.Fprintln(os.Stderr, "actions: export-env, ipv6-only, security-env, detect-hostname") - fmt.Fprintln(os.Stderr, "(also: vespa-deploy, vespa-logfmt)") - } -} - -func handleSimplePanic() { - if r := recover(); r != nil { - if jee, ok := r.(*util.JustExitError); ok { - fmt.Fprintln(os.Stderr, jee) - os.Exit(1) - } else { - panic(r) - } - } -} diff --git a/client/go/internal/admin/script-utils/services/configproxy.go b/client/go/internal/admin/script-utils/services/configproxy.go deleted file mode 100644 index 877de387efd..00000000000 --- a/client/go/internal/admin/script-utils/services/configproxy.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package services - -import ( - "fmt" - "os" - "os/exec" - "strconv" - "strings" - "time" - - "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -const ( - CONFIGPROXY_PIDFILE = "var/run/configproxy.pid" - PROXY_SERVICE_NAME = "configproxy" -) - -func JustRunConfigproxy() int { - commonPreChecks() - vespa.CheckCorrectUser() - configsources := defaults.VespaConfigserverRpcAddrs() - if len(configsources) < 1 { - util.JustExitMsg("could not find any configservers") - } - util.TuneResourceLimits() - c := jvm.NewConfigProxyJvm(PROXY_SERVICE_NAME) - userargs := os.Getenv(envvars.VESPA_CONFIGPROXY_JVMARGS) - c.ConfigureOptions(configsources, userargs) - c.Exec() - // unreachable: - return 1 -} - -func startProxyWithRunserver() { - commonPreChecks() - vespa.CheckCorrectUser() - configsources := defaults.VespaConfigserverRpcAddrs() - fmt.Printf( - "Starting config proxy using %s as config source(s)\n", - strings.Join(configsources, " and ")) - args := []string{ - "-r", "10", - "-s", PROXY_SERVICE_NAME, - "-p", CONFIGPROXY_PIDFILE, - "--", - "libexec/vespa/script-utils", "just-run-configproxy", - } - cmd := exec.Command("vespa-runserver", args...) - cmd.Stdin = nil - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Start() - p := cmd.Process - if p != nil { - p.Release() - } -} - -func waitForProxyResponse() bool { - hname, _ := vespa.FindOurHostname() - backtick := util.BackTicksWithStderr - start := time.Now() - fmt.Printf("Waiting for config proxy to start\n") - for sleepcount := 0; sleepcount < 1800; sleepcount++ { - time.Sleep(100 * time.Millisecond) - got, err := os.ReadFile(CONFIGPROXY_PIDFILE) - if err == nil { - pid, err := strconv.Atoi(strings.TrimSpace(string(got))) - if err == nil && pid > 0 { - out, err := backtick.Run("vespa-ping-configproxy", "-s", hname) - if err == nil { - secs := time.Since(start).Seconds() - fmt.Printf("config proxy started after %ds (runserver pid %d)\n", int(secs), pid) - return true - } - if sleepcount%50 == 9 { - trace.Warning("Could not ping configproxy:", err) - if sleepcount%500 == 59 { - secs := time.Since(start).Seconds() - trace.Warning("ping output after", int(secs), "seconds:", strings.TrimSpace(out)) - logFile := defaults.VespaLogFile() - cmd := fmt.Sprintf("tail -n 15 %s | vespa-logfmt -l all -N", logFile) - out, err = backtick.Run("sh", "-c", cmd) - fmt.Fprintf(os.Stderr, "tail of logfile: >>>\n%s<<<\n", out) - } - } - } else { - trace.Debug("bad contents (", string(got), ") in pid file", CONFIGPROXY_PIDFILE) - } - } else { - trace.Debug("bad pid file", CONFIGPROXY_PIDFILE, err) - } - } - secs := time.Since(start).Seconds() - fmt.Fprintf(os.Stderr, "Config proxy still failed to start after %d seconds!\n", int(secs)) - got, err := os.ReadFile(CONFIGPROXY_PIDFILE) - if err != nil { - fmt.Fprintf(os.Stderr, "pid file %s was not created\n", CONFIGPROXY_PIDFILE) - return false - } - gotpid := strings.TrimSpace(string(got)) - pid, err := strconv.Atoi(gotpid) - if err != nil || pid < 1 { - fmt.Fprintf(os.Stderr, "invalid pid '%s' in file %s\n", gotpid, CONFIGPROXY_PIDFILE) - return false - } - out, err := backtick.Run("kill", "-0", gotpid) - if err != nil { - fmt.Fprintf(os.Stderr, "config proxy process `%s` has terminated: %s\n", gotpid, strings.TrimSpace(out)) - return false - } - out, err = backtick.Run("vespa-ping-configproxy", "-s", hname) - fmt.Fprintf(os.Stderr, "failed to ping configproxy: %s\n", out) - return false -} - -func StartConfigproxy() int { - commonPreChecks() - vespa.MaybeSwitchUser("start-configproxy") - startProxyWithRunserver() - if waitForProxyResponse() { - return 0 - } - return 1 -} - -func stopProxyWithRunserver() { - _, err := util.SystemCommand.Run("vespa-runserver", - "-s", PROXY_SERVICE_NAME, - "-p", CONFIGPROXY_PIDFILE, "-S") - if err != nil { - trace.Warning("Stopping sentinel:", err) - } -} diff --git a/client/go/internal/admin/script-utils/services/env.go b/client/go/internal/admin/script-utils/services/env.go deleted file mode 100644 index 65dd583b766..00000000000 --- a/client/go/internal/admin/script-utils/services/env.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package services - -import ( - "fmt" - "os" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -func exportSettings(vespaHome string) { - vlt := fmt.Sprintf("file:%s/logs/vespa/vespa.log", vespaHome) - lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome) - dlp := fmt.Sprintf("%s/lib64:/opt/vespa-deps/lib64", vespaHome) - os.Setenv(envvars.VESPA_LOG_TARGET, vlt) - os.Setenv(envvars.VESPA_LOG_CONTROL_DIR, lcd) - os.Setenv(envvars.LD_LIBRARY_PATH, dlp) - os.Setenv(envvars.JAVAVM_LD_PRELOAD, "") - os.Setenv(envvars.LD_PRELOAD, "") - os.Setenv(envvars.MALLOC_ARENA_MAX, "1") - util.OptionallyReduceTimerFrequency() -} diff --git a/client/go/internal/admin/script-utils/services/prechecks.go b/client/go/internal/admin/script-utils/services/prechecks.go deleted file mode 100644 index 690447a9fee..00000000000 --- a/client/go/internal/admin/script-utils/services/prechecks.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package services - -import ( - "os" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func commonPreChecks() (veHome, veHost string) { - if doTrace := os.Getenv(envvars.TRACE_STARTUP); doTrace != "" { - trace.AdjustVerbosity(1) - } - if doDebug := os.Getenv(envvars.DEBUG_STARTUP); doDebug != "" { - trace.AdjustVerbosity(2) - } - _ = vespa.FindAndVerifyVespaHome() - err := vespa.LoadDefaultEnv() - if err != nil { - panic(err) - } - veHome = vespa.FindAndVerifyVespaHome() - veHost, err = vespa.FindOurHostname() - if err != nil { - trace.Warning("could not detect hostname:", err, "; using fallback:", veHost) - } - err = os.Chdir(veHome) - if err != nil { - util.JustExitWith(err) - } - return -} diff --git a/client/go/internal/admin/script-utils/services/sentinel.go b/client/go/internal/admin/script-utils/services/sentinel.go deleted file mode 100644 index 1017924263b..00000000000 --- a/client/go/internal/admin/script-utils/services/sentinel.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package services - -import ( - "fmt" - "os" - "os/exec" - "strconv" - "strings" - "time" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -const ( - SENTINEL_PIDFILE = "var/run/sentinel.pid" - SENTINEL_SERVICE_NAME = "config-sentinel" -) - -func startSentinelWithRunserver() { - _, veHost := commonPreChecks() - vespa.CheckCorrectUser() - os.Setenv(envvars.VESPA_SERVICE_NAME, SENTINEL_SERVICE_NAME) - configId := fmt.Sprintf("hosts/%s", veHost) - args := []string{ - "-r", "10", - "-s", SENTINEL_SERVICE_NAME, - "-p", SENTINEL_PIDFILE, - "--", - "sbin/vespa-config-sentinel", - "-c", configId, - } - cmd := exec.Command("vespa-runserver", args...) - cmd.Stdin = nil - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Start() - p := cmd.Process - if p != nil { - p.Release() - } - os.Unsetenv(envvars.VESPA_SERVICE_NAME) -} - -func waitForSentinelPid() bool { - backtick := util.BackTicksWithStderr - start := time.Now() - for sleepcount := 0; sleepcount < 1000; sleepcount++ { - time.Sleep(10 * time.Millisecond) - got, err := os.ReadFile(CONFIGPROXY_PIDFILE) - if err == nil { - pid, err := strconv.Atoi(strings.TrimSpace(string(got))) - if err == nil && pid > 0 { - _, err := backtick.Run("kill", "-0", strconv.Itoa(pid)) - if err == nil { - secs := int(time.Since(start).Seconds()) - if secs > 0 { - fmt.Printf("config sentinel started after %d seconds\n", secs) - } - return true - } - } - } - if sleepcount%500 == 90 { - trace.Warning("Waiting for sentinel to start") - } - } - return false -} - -func StartConfigSentinel() int { - commonPreChecks() - vespa.MaybeSwitchUser("start-config-sentinel") - startSentinelWithRunserver() - if waitForSentinelPid() { - return 0 - } - return 1 -} - -func stopSentinelWithRunserver() { - _, err := util.SystemCommand.Run("vespa-runserver", - "-s", SENTINEL_SERVICE_NAME, - "-p", SENTINEL_PIDFILE, "-S") - if err != nil { - trace.Warning("Stopping sentinel:", err) - } -} diff --git a/client/go/internal/admin/script-utils/services/start.go b/client/go/internal/admin/script-utils/services/start.go deleted file mode 100644 index f47d99714f1..00000000000 --- a/client/go/internal/admin/script-utils/services/start.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package services - -import ( - "fmt" - "os" - "strings" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func StartServices() int { - veHome, _ := commonPreChecks() - vespa.CheckCorrectUser() - trace.Debug("running as correct user") - exportSettings(veHome) - if vespa.HasOnlyIpV6() { - os.Setenv(envvars.VESPA_ONLY_IP_V6_NETWORKING, "true") - } - startProxyWithRunserver() - if waitForProxyResponse() { - startSentinelWithRunserver() - if waitForSentinelPid() { - return 0 - } - } - return 1 -} - -func checkjava() { - backticks := util.BackTicksWithStderr - out, err := backticks.Run("java", "-version") - if err != nil { - trace.Warning("cannot run 'java -version'") - util.JustExitWith(err) - } - if !strings.Contains(out, "64-Bit Server VM") { - util.JustExitWith(fmt.Errorf("java must invoke the 64-bit Java VM, but -version says:\n%s\n", out)) - } -} - -func runvalidation() { - // not implemented -} - -func VespaStartServices() int { - home, host := commonPreChecks() - trace.Debug("common prechecks ok, running in", home, "on", host) - vespa.RunPreStart() - trace.Debug("prestart ok") - util.TuneResourceLimits() - trace.Debug("resource limits ok") - checkjava() - trace.Debug("java ok") - runvalidation() - enable_transparent_hugepages_with_background_compaction() - disable_vm_zone_reclaim_mode() - drop_caches() - err := vespa.MaybeSwitchUser("start-services") - if err != nil { - util.JustExitWith(err) - } - return StartServices() -} diff --git a/client/go/internal/admin/script-utils/services/stop.go b/client/go/internal/admin/script-utils/services/stop.go deleted file mode 100644 index f5b764d122e..00000000000 --- a/client/go/internal/admin/script-utils/services/stop.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package services - -import ( - "os" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func VespaStopServices() int { - if doTrace := os.Getenv(envvars.TRACE_STARTUP); doTrace != "" { - trace.AdjustVerbosity(1) - } - if doDebug := os.Getenv(envvars.DEBUG_STARTUP); doDebug != "" { - trace.AdjustVerbosity(2) - } - err := vespa.LoadDefaultEnv() - if err != nil { - util.JustExitWith(err) - } - err = vespa.MaybeSwitchUser("vespa-stop-services") - if err != nil { - util.JustExitWith(err) - } - vespa.CheckCorrectUser() - trace.Debug("running as correct user") - stopSentinelWithRunserver() - stopProxyWithRunserver() - return 0 -} diff --git a/client/go/internal/admin/script-utils/services/tuning.go b/client/go/internal/admin/script-utils/services/tuning.go deleted file mode 100644 index f922495812f..00000000000 --- a/client/go/internal/admin/script-utils/services/tuning.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package services - -import ( - "fmt" - "os" - - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" -) - -const ( - DROP_CACHES_CTL = "/proc/sys/vm/drop_caches" - TRANSPARENT_HUGEPAGE_ENABLED = "/sys/kernel/mm/transparent_hugepage/enabled" - TRANSPARENT_HUGEPAGE_DEFRAG = "/sys/kernel/mm/transparent_hugepage/defrag" - TRANSPARENT_HUGEPAGE_KH_DEFRAG = "/sys/kernel/mm/transparent_hugepage/khugepaged/defrag" - ZONE_RECLAIM_CTL = "/proc/sys/vm/zone_reclaim_mode" -) - -func maybeEcho(fileName, toWrite string) bool { - f, err := os.OpenFile(fileName, os.O_WRONLY, 0) - if err == nil { - _, err = fmt.Fprintf(f, "%s\n", toWrite) - f.Close() - if err == nil { - trace.Debug("wrote", toWrite, "to", fileName) - return true - } else { - trace.Warning(err) - } - } - return false -} - -func enable_transparent_hugepages_with_background_compaction() { - // Should probably also be done on host. - maybeEcho(TRANSPARENT_HUGEPAGE_ENABLED, "always") - maybeEcho(TRANSPARENT_HUGEPAGE_DEFRAG, "never") - maybeEcho(TRANSPARENT_HUGEPAGE_KH_DEFRAG, "1") -} - -func disable_vm_zone_reclaim_mode() { - maybeEcho(ZONE_RECLAIM_CTL, "0") -} - -func drop_caches() { - if maybeEcho(DROP_CACHES_CTL, "3") { - trace.Debug("dropped caches") - } -} diff --git a/client/go/internal/admin/script-utils/standalone/start.go b/client/go/internal/admin/script-utils/standalone/start.go deleted file mode 100644 index a26ac8e9d8c..00000000000 --- a/client/go/internal/admin/script-utils/standalone/start.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -// for starting standalone jdisc containers -package standalone - -import ( - "os" - - "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func commonPreChecks() { - if doTrace := os.Getenv("TRACE_JVM_STARTUP"); doTrace != "" { - trace.AdjustVerbosity(1) - } - if doDebug := os.Getenv("DEBUG_JVM_STARTUP"); doDebug != "" { - trace.AdjustVerbosity(2) - } - veHome := vespa.FindAndVerifyVespaHome() - err := os.Chdir(veHome) - if err != nil { - util.JustExitWith(err) - } - err = vespa.LoadDefaultEnv() - if err != nil { - util.JustExitWith(err) - } -} - -func StartStandaloneContainer(extraArgs []string) int { - commonPreChecks() - util.TuneResourceLimits() - serviceName := os.Getenv("VESPA_SERVICE_NAME") - if serviceName == "" { - util.JustExitMsg("Missing service name, ensure VESPA_SERVICE_NAME is set in the environment") - } - c := jvm.NewStandaloneContainer(serviceName) - jvmOpts := c.JvmOptions() - jvmOpts.AddOption("-DOnnxBundleActivator.skip=true") - for _, extra := range extraArgs { - jvmOpts.AddOption(extra) - } - minFallback := jvm.MegaBytesOfMemory(128) - maxFallback := jvm.MegaBytesOfMemory(2048) - jvmOpts.AddDefaultHeapSizeArgs(minFallback, maxFallback) - c.Exec() - // unreachable: - return 1 -} diff --git a/client/go/internal/admin/script-utils/startcbinary/cmd.go b/client/go/internal/admin/script-utils/startcbinary/cmd.go deleted file mode 100644 index 9580a9240bb..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/cmd.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package startcbinary - -import ( - "os" - "strings" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func Run(args []string) int { - trace.AdjustVerbosity(0) - if len(args) < 1 { - trace.Warning("missing program argument") - return 1 - } - spec := NewProgSpec(args) - err := vespa.LoadDefaultEnv() - if err != nil { - util.JustExitWith(err) - } - hostname, err := vespa.FindOurHostname() - if err != nil { - trace.Warning("could not detect hostname:", err, "; using fallback:", hostname) - } - return startCbinary(spec) -} - -func IsCandidate(program string) bool { - binary := program + "-bin" - if strings.Contains(binary, "/") { - return util.IsRegularFile(binary) - } else { - path := strings.Split(os.Getenv(envvars.PATH), ":") - for _, dir := range path { - fn := dir + "/" + binary - if util.IsRegularFile(fn) { - return true - } - } - } - return false -} diff --git a/client/go/internal/admin/script-utils/startcbinary/common_env.go b/client/go/internal/admin/script-utils/startcbinary/common_env.go deleted file mode 100644 index 6bc730b5119..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/common_env.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package startcbinary - -import ( - "os" - "strings" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func (spec *ProgSpec) considerFallback(varName, varValue string) { - if spec.getenv(varName) == "" && varValue != "" { - spec.setenv(varName, varValue) - } -} - -func (spec *ProgSpec) considerEnvFallback(targetVar, fallbackVar string) { - spec.considerFallback(targetVar, spec.getenv(fallbackVar)) -} - -func (spec *ProgSpec) configureCommonEnv() { - os.Unsetenv(envvars.LD_PRELOAD) - spec.setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") - spec.setenv(envvars.GLIBCXX_FORCE_NEW, "1") - spec.setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") - spec.setenv(envvars.MALLOC_ARENA_MAX, "1") - - // fallback from old env.vars: - spec.considerEnvFallback(envvars.VESPA_USE_HUGEPAGES_LIST, envvars.HUGEPAGES_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_MADVISE_LIST, envvars.MADVISE_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC, envvars.VESPAMALLOC_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_D, envvars.VESPAMALLOCD_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_DST, envvars.VESPAMALLOCDST_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_NO_VESPAMALLOC, envvars.NO_VESPAMALLOC_LIST) - // other fallbacks: - spec.considerFallback(envvars.ROOT, vespa.FindHome()) - spec.considerFallback(envvars.VESPA_USER, vespa.FindVespaUser()) - spec.considerFallback(envvars.VESPA_USE_HUGEPAGES_LIST, "all") - spec.considerFallback(envvars.VESPA_USE_VESPAMALLOC, "all") - spec.considerFallback(envvars.VESPA_USE_NO_VESPAMALLOC, strings.Join([]string{ - "vespa-rpc-invoke", - "vespa-get-config", - "vespa-sentinel-cmd", - "vespa-route", - "vespa-proton-cmd", - "vespa-configproxy-cmd", - "vespa-config-status", - }, " ")) - -} - -func (spec *ProgSpec) configureHugePages() { - if spec.matchesListEnv(envvars.VESPA_USE_HUGEPAGES_LIST) { - spec.setenv(envvars.VESPA_USE_HUGEPAGES, "yes") - } -} - -func (spec *ProgSpec) configureUseMadvise() { - limit := spec.valueFromListEnv(envvars.VESPA_USE_MADVISE_LIST) - if limit != "" { - trace.Trace("shall use madvise with limit", limit, "as set in", envvars.VESPA_USE_MADVISE_LIST) - spec.setenv(envvars.VESPA_MALLOC_MADVISE_LIMIT, limit) - return - } -} - -func (spec *ProgSpec) configurePath() { - // Prefer newer gdb and pstack: - spec.prependPath("/opt/rh/gcc-toolset-12/root/usr/bin") - // Maven is needed for tester applications: - spec.prependPath(vespa.FindHome() + "/local/maven/bin") - spec.prependPath(vespa.FindHome() + "/bin64") - spec.prependPath(vespa.FindHome() + "/bin") - // how to find the "java" program? - // should be available in $VESPA_HOME/bin or JAVA_HOME - if javaHome := spec.getenv(envvars.JAVA_HOME); javaHome != "" { - spec.prependPath(javaHome + "/bin") - } -} diff --git a/client/go/internal/admin/script-utils/startcbinary/mockbin/bad-numactl b/client/go/internal/admin/script-utils/startcbinary/mockbin/bad-numactl deleted file mode 100755 index 1b08bb19b95..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/mockbin/bad-numactl +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -# emulate numactl which says hardware is available but fails anyway - -case $#.$* in - 1.--hardware) - cat << 'EOF' -available: 2 nodes (0-1) -node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 -node 0 size: 128604 MB -node 0 free: 107167 MB -node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39 41 43 45 47 -node 1 size: 128976 MB -node 1 free: 110921 MB -node distances: -node 0 1 - 0: 10 21 - 1: 21 10 -EOF - ;; - *) - echo "numactl failure" - ;; -esac -exit 0 diff --git a/client/go/internal/admin/script-utils/startcbinary/mockbin/good-numactl b/client/go/internal/admin/script-utils/startcbinary/mockbin/good-numactl deleted file mode 100755 index f861809b382..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/mockbin/good-numactl +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -# emulate normal numactl operations - -case $#.$* in - 1.--hardware) - cat << 'EOF' -available: 2 nodes (0-1) -node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 -node 0 size: 128604 MB -node 0 free: 107167 MB -node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39 41 43 45 47 -node 1 size: 128976 MB -node 1 free: 110921 MB -node distances: -node 0 1 - 0: 10 21 - 1: 21 10 -EOF - ;; - *) - echo "foo" - ;; -esac -exit 0 diff --git a/client/go/internal/admin/script-utils/startcbinary/mockbin/has-valgrind b/client/go/internal/admin/script-utils/startcbinary/mockbin/has-valgrind deleted file mode 100755 index 8716209928e..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/mockbin/has-valgrind +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -# emulate "which" that finds valgrind - -echo "/opt/vespa-deps/bin/valgrind" -exit 0 diff --git a/client/go/internal/admin/script-utils/startcbinary/mockbin/no-numactl b/client/go/internal/admin/script-utils/startcbinary/mockbin/no-numactl deleted file mode 100755 index dfe583abaca..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/mockbin/no-numactl +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -# emulate inoperational numactl - -echo "No NUMA available on this system" -exit 1 diff --git a/client/go/internal/admin/script-utils/startcbinary/mockbin/no-valgrind b/client/go/internal/admin/script-utils/startcbinary/mockbin/no-valgrind deleted file mode 100755 index 7a74e29b84b..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/mockbin/no-valgrind +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -# emulate "which" that does not find valgrind - -echo "which: no valgrind in ($PATH)" >&2 -exit 1 diff --git a/client/go/internal/admin/script-utils/startcbinary/numactl.go b/client/go/internal/admin/script-utils/startcbinary/numactl.go deleted file mode 100644 index fe091dedba9..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/numactl.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package startcbinary - -import ( - "fmt" - "strconv" - "strings" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -func (p *ProgSpec) configureNumaCtl() { - p.shouldUseNumaCtl = false - p.numaSocket = -1 - if p.getenv(envvars.VESPA_NO_NUMACTL) != "" { - return - } - backticks := util.BackTicksIgnoreStderr - out, err := backticks.Run("numactl", "--hardware") - trace.Debug("numactl --hardware says:", out) - if err != nil { - trace.Trace("numactl error:", err) - return - } - outfoo, errfoo := backticks.Run("numactl", "--interleave", "all", "echo", "foo") - if errfoo != nil { - trace.Trace("cannot run with numactl:", errfoo) - return - } - if outfoo != "foo\n" { - trace.Trace("bad numactl output:", outfoo) - return - } - p.shouldUseNumaCtl = true - if affinity := p.getenv(envvars.VESPA_AFFINITY_CPU_SOCKET); affinity != "" { - wantSocket, _ := strconv.Atoi(affinity) - trace.Debug("want socket:", wantSocket) - parts := strings.Fields(out) - for idx := 0; idx+2 < len(parts); idx++ { - if parts[idx] == "available:" && parts[idx+2] == "nodes" { - numSockets, _ := strconv.Atoi(parts[idx+1]) - trace.Debug("numSockets:", numSockets) - if numSockets > 1 { - p.numaSocket = (wantSocket % numSockets) - return - } - } - } - } -} - -func (p *ProgSpec) numaCtlBinary() string { - return "numactl" -} - -func (p *ProgSpec) prependNumaCtl(args []string) []string { - v := util.NewArrayList[string](5 + len(args)) - v.Append("numactl") - if p.numaSocket >= 0 { - v.Append(fmt.Sprintf("--cpunodebind=%d", p.numaSocket)) - v.Append(fmt.Sprintf("--membind=%d", p.numaSocket)) - } else { - v.Append("--interleave") - v.Append("all") - } - v.AppendAll(args...) - return v -} diff --git a/client/go/internal/admin/script-utils/startcbinary/numactl_test.go b/client/go/internal/admin/script-utils/startcbinary/numactl_test.go deleted file mode 100644 index 65f52be988e..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/numactl_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package startcbinary - -import ( - "fmt" - "os" - "runtime" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" -) - -func setup(t *testing.T, testFileName string) { - trace.AdjustVerbosity(1) - mockBinParent = strings.TrimSuffix(testFileName, "/numactl_test.go") - tmpBin = t.TempDir() + "/mock.bin.numactl_test" - err := os.MkdirAll(tmpBin, 0755) - assert.Nil(t, err) - t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH"))) -} - -func TestNumaCtlDetection(t *testing.T) { - if runtime.GOOS == "windows" { - return - } - _, tfn, _, _ := runtime.Caller(0) - setup(t, tfn) - orig := []string{"/bin/myprog", "-c", "cfgid"} - spec := NewProgSpec(orig) - - useMock("no-numactl", "numactl") - spec.configureNumaCtl() - assert.Equal(t, false, spec.shouldUseNumaCtl) - - useMock("bad-numactl", "numactl") - spec.configureNumaCtl() - assert.Equal(t, false, spec.shouldUseNumaCtl) - - t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "") - useMock("good-numactl", "numactl") - spec.configureNumaCtl() - assert.Equal(t, true, spec.shouldUseNumaCtl) - assert.Equal(t, -1, spec.numaSocket) - argv := spec.prependNumaCtl(orig) - trace.Trace("argv:", argv) - assert.Equal(t, 6, len(argv)) - assert.Equal(t, "numactl", argv[0]) - assert.Equal(t, "--interleave", argv[1]) - assert.Equal(t, "all", argv[2]) - assert.Equal(t, "/bin/myprog-bin", argv[3]) - assert.Equal(t, "-c", argv[4]) - assert.Equal(t, "cfgid", argv[5]) - - t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "0") - spec.configureNumaCtl() - assert.Equal(t, true, spec.shouldUseNumaCtl) - assert.Equal(t, 0, spec.numaSocket) - argv = spec.prependNumaCtl(orig) - trace.Trace("argv:", argv) - assert.Equal(t, 6, len(argv)) - assert.Equal(t, "numactl", argv[0]) - assert.Equal(t, "--cpunodebind=0", argv[1]) - assert.Equal(t, "--membind=0", argv[2]) - assert.Equal(t, "/bin/myprog-bin", argv[3]) - assert.Equal(t, "-c", argv[4]) - assert.Equal(t, "cfgid", argv[5]) - - t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "1") - spec.configureNumaCtl() - assert.Equal(t, true, spec.shouldUseNumaCtl) - assert.Equal(t, 1, spec.numaSocket) - argv = spec.prependNumaCtl(orig) - trace.Trace("argv:", argv) - assert.Equal(t, 6, len(argv)) - assert.Equal(t, "numactl", argv[0]) - assert.Equal(t, "--cpunodebind=1", argv[1]) - assert.Equal(t, "--membind=1", argv[2]) - assert.Equal(t, "/bin/myprog-bin", argv[3]) - assert.Equal(t, "-c", argv[4]) - assert.Equal(t, "cfgid", argv[5]) - - t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "2") - spec.configureNumaCtl() - assert.Equal(t, true, spec.shouldUseNumaCtl) - assert.Equal(t, 0, spec.numaSocket) - -} diff --git a/client/go/internal/admin/script-utils/startcbinary/progspec.go b/client/go/internal/admin/script-utils/startcbinary/progspec.go deleted file mode 100644 index 9975f6c3c90..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/progspec.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package startcbinary - -import ( - "os" - "strings" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" -) - -type ProgSpec struct { - Program string - Args []string - BaseName string - Env map[string]string - numaSocket int - shouldUseCallgrind bool - shouldUseValgrind bool - shouldUseNumaCtl bool - shouldUseVespaMalloc bool - vespaMallocPreload string -} - -func NewProgSpec(argv []string) *ProgSpec { - progName := argv[0] - binProg := progName + "-bin" - p := ProgSpec{ - Program: binProg, - Args: argv, - BaseName: baseNameOf(progName), - Env: make(map[string]string), - numaSocket: -1, - } - p.Args[0] = binProg - return &p -} - -func baseNameOf(s string) string { - idx := strings.LastIndex(s, "/") - idx++ - return s[idx:] -} - -func (p *ProgSpec) setenv(k, v string) { - p.Env[k] = v -} - -func (p *ProgSpec) getenv(k string) string { - if v, ok := p.Env[k]; ok { - return v - } - return os.Getenv(k) -} - -func (p *ProgSpec) prependPath(dirName string) { - pathList := []string{dirName} - oldPath := p.getenv(envvars.PATH) - if oldPath == "" { - oldPath = "/usr/bin" - } - for _, part := range strings.Split(oldPath, ":") { - if part != dirName { - pathList = append(pathList, part) - } - } - newPath := strings.Join(pathList, ":") - p.setenv(envvars.PATH, newPath) - os.Setenv(envvars.PATH, newPath) -} - -func (p *ProgSpec) matchesListEnv(envVarName string) bool { - return p.matchesListString(p.getenv(envVarName)) -} - -func (p *ProgSpec) matchesListString(env string) bool { - if env == "all" { - trace.Debug(p.Program, "always matching in:", env) - return true - } - parts := strings.Fields(env) - for _, part := range parts { - if p.BaseName == part { - trace.Debug(p.Program, "has basename matching in:", env) - return true - } - trace.Debug("checking matching:", p.BaseName, "!=", part) - } - return false -} - -func (p *ProgSpec) valueFromListEnv(envVarName string) string { - return p.valueFromListString(p.getenv(envVarName)) -} - -func (p *ProgSpec) valueFromListString(env string) string { - parts := strings.Fields(env) - for _, part := range parts { - idx := strings.Index(part, "=") - if idx <= 0 { - trace.Trace("expected key=value, but got:", part) - continue - } - partName := part[:idx] - idx++ - partValue := part[idx:] - if p.BaseName == partName || partName == "all" { - trace.Debug(p.Program, "has basename matching in:", env) - return partValue - } - trace.Debug("checking matching:", p.BaseName, "!=", part) - } - return "" -} - -func (spec *ProgSpec) effectiveEnv() []string { - env := make(map[string]string) - for _, entry := range os.Environ() { - addInMap := func(kv string) bool { - for idx, elem := range kv { - if elem == '=' { - k := kv[:idx] - env[k] = kv - return true - } - } - return false - } - if !addInMap(entry) { - env[entry] = "" - } - } - for k, v := range spec.Env { - trace.Trace("add to environment:", k, "=", v) - env[k] = k + "=" + v - } - envv := make([]string, 0, len(env)) - for _, v := range env { - envv = append(envv, v) - } - return envv -} diff --git a/client/go/internal/admin/script-utils/startcbinary/progspec_test.go b/client/go/internal/admin/script-utils/startcbinary/progspec_test.go deleted file mode 100644 index be113e4e350..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/progspec_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package startcbinary - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestProgSpec(t *testing.T) { - spec := NewProgSpec([]string{"/opt/vespa/bin/foobar"}) - var b bool - - b = spec.matchesListString("") - assert.Equal(t, false, b) - b = spec.matchesListString("foobar") - assert.Equal(t, true, b) - b = spec.matchesListString("foo bar") - assert.Equal(t, false, b) - b = spec.matchesListString("one foobar") - assert.Equal(t, true, b) - b = spec.matchesListString("foobar two") - assert.Equal(t, true, b) - b = spec.matchesListString("one foobar two") - assert.Equal(t, true, b) - b = spec.matchesListString("all") - assert.Equal(t, true, b) - - var s string - s = spec.valueFromListString("") - assert.Equal(t, "", s) - s = spec.valueFromListString("foobar=123") - assert.Equal(t, "123", s) - s = spec.valueFromListString("one=1 foobar=123 two=2") - assert.Equal(t, "123", s) - s = spec.valueFromListString("one=1 all=123") - assert.Equal(t, "123", s) -} diff --git a/client/go/internal/admin/script-utils/startcbinary/startcbinary.go b/client/go/internal/admin/script-utils/startcbinary/startcbinary.go deleted file mode 100644 index f5e58e59808..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/startcbinary.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package startcbinary - -import ( - "fmt" - "os" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -func startCbinary(spec *ProgSpec) int { - spec.configureCommonEnv() - spec.configurePath() - spec.configureTuning() - spec.configureValgrind() - spec.configureNumaCtl() - spec.configureHugePages() - spec.configureUseMadvise() - spec.configureVespaMalloc() - err := spec.run() - if err != nil { - fmt.Fprintln(os.Stderr, err) - return 1 - } else { - return 0 - } -} - -func (spec *ProgSpec) run() error { - prog := spec.Program - args := spec.Args - if spec.shouldUseValgrind { - args = spec.prependValgrind(args) - prog = spec.valgrindBinary() - } else if spec.shouldUseNumaCtl { - args = spec.prependNumaCtl(args) - prog = spec.numaCtlBinary() - } - if spec.shouldUseVespaMalloc { - spec.setenv(envvars.LD_PRELOAD, spec.vespaMallocPreload) - } - envv := spec.effectiveEnv() - return util.Execvpe(prog, args, envv) -} diff --git a/client/go/internal/admin/script-utils/startcbinary/tuning.go b/client/go/internal/admin/script-utils/startcbinary/tuning.go deleted file mode 100644 index 57230629d7a..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/tuning.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package startcbinary - -import ( - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -func (spec *ProgSpec) configureTuning() { - util.OptionallyReduceTimerFrequency() - util.TuneResourceLimits() -} diff --git a/client/go/internal/admin/script-utils/startcbinary/valgrind.go b/client/go/internal/admin/script-utils/startcbinary/valgrind.go deleted file mode 100644 index 43a1ed602bd..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/valgrind.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package startcbinary - -import ( - "fmt" - "os" - "strings" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func (p *ProgSpec) configureValgrind() { - p.shouldUseValgrind = false - p.shouldUseCallgrind = false - env := p.getenv(envvars.VESPA_USE_VALGRIND) - parts := strings.Split(env, " ") - for _, part := range parts { - if p.BaseName == part { - trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env) - backticks := util.BackTicksWithStderr - out, err := backticks.Run("which", "valgrind") - if err != nil { - trace.Trace("no valgrind, 'which' fails:", err, "=>", out) - return - } - if opts := p.getenv(envvars.VESPA_VALGRIND_OPT); strings.Contains(opts, "callgrind") { - p.shouldUseCallgrind = true - } - p.shouldUseValgrind = true - return - } - trace.Debug("checking", envvars.VESPA_USE_VALGRIND, ":", p.BaseName, "!=", part) - } -} - -func (p *ProgSpec) valgrindBinary() string { - return "valgrind" -} - -func (p *ProgSpec) valgrindOptions() []string { - env := p.getenv(envvars.VESPA_VALGRIND_OPT) - if env != "" { - return strings.Fields(env) - } - result := []string{ - "--num-callers=32", - "--run-libc-freeres=yes", - "--track-origins=yes", - "--freelist-vol=1000000000", - "--leak-check=full", - "--show-reachable=yes", - } - result = addValgrindSuppression(result, "etc/vespa/valgrind-suppressions.txt") - result = addValgrindSuppression(result, "etc/vespa/suppressions.txt") - return result -} - -func addValgrindSuppression(r []string, fn string) []string { - existsOk, fileName := vespa.HasFileUnderVespaHome(fn) - if existsOk { - r = append(r, fmt.Sprintf("--suppressions=%s", fileName)) - } - return r -} - -func (p *ProgSpec) valgrindLogOption() string { - return fmt.Sprintf("--log-file=%s/tmp/valgrind.%s.log.%d", vespa.FindHome(), p.BaseName, os.Getpid()) -} - -func (p *ProgSpec) prependValgrind(args []string) []string { - v := util.NewArrayList[string](15 + len(args)) - v.Append(p.valgrindBinary()) - v.AppendAll(p.valgrindOptions()...) - v.Append(p.valgrindLogOption()) - v.AppendAll(args...) - return v -} diff --git a/client/go/internal/admin/script-utils/startcbinary/valgrind_test.go b/client/go/internal/admin/script-utils/startcbinary/valgrind_test.go deleted file mode 100644 index 48cc78474ed..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/valgrind_test.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package startcbinary - -import ( - "fmt" - "os" - "runtime" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -var tmpBin string -var mockBinParent string - -func useMock(prog, target string) { - mock := fmt.Sprintf("%s/mockbin/%s", mockBinParent, prog) - symlink := fmt.Sprintf("%s/%s", tmpBin, target) - os.Remove(symlink) - err := os.Symlink(mock, symlink) - if err != nil { - util.JustExitWith(err) - } -} - -func setupValgrind(t *testing.T, testFileName string) { - trace.AdjustVerbosity(1) - t.Setenv("VESPA_HOME", mockBinParent+"/mock_vespahome") - mockBinParent = strings.TrimSuffix(testFileName, "/valgrind_test.go") - tmpBin = t.TempDir() + "/mock.bin.valgrind_test" - err := os.MkdirAll(tmpBin, 0755) - assert.Nil(t, err) - t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH"))) -} - -func TestValgrindDetection(t *testing.T) { - if runtime.GOOS == "windows" { - return - } - _, tfn, _, _ := runtime.Caller(0) - setupValgrind(t, tfn) - spec := NewProgSpec([]string{"/opt/vespa/bin/foobar"}) - var argv []string - - useMock("has-valgrind", "which") - - t.Setenv("VESPA_USE_VALGRIND", "") - spec.configureValgrind() - assert.Equal(t, false, spec.shouldUseValgrind) - assert.Equal(t, false, spec.shouldUseCallgrind) - - t.Setenv("VESPA_USE_VALGRIND", "foo bar") - spec.configureValgrind() - assert.Equal(t, false, spec.shouldUseValgrind) - assert.Equal(t, false, spec.shouldUseCallgrind) - - t.Setenv("VESPA_USE_VALGRIND", "foobar") - spec.configureValgrind() - assert.Equal(t, true, spec.shouldUseValgrind) - assert.Equal(t, false, spec.shouldUseCallgrind) - - argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"}) - trace.Trace("argv:", argv) - assert.Equal(t, 11, len(argv)) - assert.Equal(t, "valgrind", argv[0]) - assert.Equal(t, "/bin/myprog", argv[8]) - - t.Setenv("VESPA_USE_VALGRIND", "another foobar yetmore") - spec.configureValgrind() - assert.Equal(t, true, spec.shouldUseValgrind) - assert.Equal(t, false, spec.shouldUseCallgrind) - - t.Setenv("VESPA_VALGRIND_OPT", "--tool=callgrind") - spec.configureValgrind() - assert.Equal(t, true, spec.shouldUseValgrind) - assert.Equal(t, true, spec.shouldUseCallgrind) - - argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"}) - trace.Trace("argv:", argv) - assert.Equal(t, 6, len(argv)) - assert.Equal(t, "valgrind", argv[0]) - assert.Equal(t, "--tool=callgrind", argv[1]) - assert.Equal(t, "/bin/myprog", argv[3]) - - useMock("no-valgrind", "which") - spec.configureValgrind() - assert.Equal(t, false, spec.shouldUseValgrind) - assert.Equal(t, false, spec.shouldUseCallgrind) -} diff --git a/client/go/internal/admin/script-utils/startcbinary/vespamalloc.go b/client/go/internal/admin/script-utils/startcbinary/vespamalloc.go deleted file mode 100644 index c6d53e1d03c..00000000000 --- a/client/go/internal/admin/script-utils/startcbinary/vespamalloc.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package startcbinary - -import ( - "fmt" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func vespaMallocLib(suf string) string { - prefixes := []string{"lib64", "lib"} - for _, pre := range prefixes { - fn := fmt.Sprintf("%s/vespa/malloc/%s", pre, suf) - existsOk, fileName := vespa.HasFileUnderVespaHome(fn) - if existsOk { - trace.Debug("found library:", fileName) - return fileName - } - trace.Debug("bad or missing library:", fn) - } - return "" -} - -func (p *ProgSpec) configureVespaMalloc() { - p.shouldUseVespaMalloc = false - if p.matchesListEnv(envvars.VESPA_USE_NO_VESPAMALLOC) { - trace.Trace("use no vespamalloc:", p.BaseName) - return - } - if p.shouldUseValgrind && !p.shouldUseCallgrind { - trace.Trace("use valgrind, so no vespamalloc:", p.BaseName) - return - } - var useFile string - if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_DST) { - useFile = vespaMallocLib("libvespamallocdst16.so") - } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_D) { - useFile = vespaMallocLib("libvespamallocd.so") - } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC) { - useFile = vespaMallocLib("libvespamalloc.so") - } - trace.Trace("use file:", useFile) - if useFile == "" { - return - } - if loadAsHuge := p.getenv(envvars.VESPA_LOAD_CODE_AS_HUGEPAGES); loadAsHuge != "" { - otherFile := vespaMallocLib("libvespa_load_as_huge.so") - useFile = fmt.Sprintf("%s:%s", useFile, otherFile) - } - p.considerEnvFallback(envvars.VESPA_MALLOC_HUGEPAGES, envvars.VESPA_USE_HUGEPAGES) - p.vespaMallocPreload = useFile - p.shouldUseVespaMalloc = true -} diff --git a/client/go/internal/admin/vespa-wrapper/configserver/check.go b/client/go/internal/admin/vespa-wrapper/configserver/check.go new file mode 100644 index 00000000000..a0248dd128f --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/configserver/check.go @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func checkIsConfigserver(myname string) { + onlyHosts := defaults.VespaConfigserverHosts() + for _, hn := range onlyHosts { + if hn == "localhost" || hn == myname { + trace.Debug("should run configserver:", hn) + return + } + } + trace.Warning("only these hosts should run a config server:", onlyHosts) + util.JustExitMsg(fmt.Sprintf("this host [%s] should not run a config server", myname)) +} diff --git a/client/go/internal/admin/vespa-wrapper/configserver/env.go b/client/go/internal/admin/vespa-wrapper/configserver/env.go new file mode 100644 index 00000000000..1c4c33f5628 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/configserver/env.go @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func exportSettings(vespaHome string) { + vlt := fmt.Sprintf("file:%s/logs/vespa/vespa.log", vespaHome) + lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome) + lcf := fmt.Sprintf("%s/configserver.logcontrol", lcd) + dlp := fmt.Sprintf("%s/lib64", vespaHome) + app := fmt.Sprintf("%s/conf/configserver-app", vespaHome) + os.Setenv(envvars.VESPA_LOG_TARGET, vlt) + os.Setenv(envvars.VESPA_LOG_CONTROL_DIR, lcd) + os.Setenv(envvars.VESPA_LOG_CONTROL_FILE, lcf) + os.Setenv(envvars.VESPA_SERVICE_NAME, "configserver") + os.Setenv(envvars.LD_LIBRARY_PATH, dlp) + os.Setenv(envvars.JAVAVM_LD_PRELOAD, "") + os.Setenv(envvars.LD_PRELOAD, "") + os.Setenv(envvars.STANDALONE_JDISC_APP_LOCATION, app) + os.Setenv(envvars.STANDALONE_JDISC_DEPLOYMENT_PROFILE, "configserver") + os.Setenv(envvars.MALLOC_ARENA_MAX, "1") + util.OptionallyReduceTimerFrequency() +} diff --git a/client/go/internal/admin/vespa-wrapper/configserver/fix_dirs_and_files.go b/client/go/internal/admin/vespa-wrapper/configserver/fix_dirs_and_files.go new file mode 100644 index 00000000000..8410d3f657c --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/configserver/fix_dirs_and_files.go @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func makeFixSpec() util.FixSpec { + vespaUid, vespaGid := vespa.FindVespaUidAndGid() + return util.FixSpec{ + UserId: vespaUid, + GroupId: vespaGid, + DirMode: 0755, + FileMode: 0644, + } +} + +func fixDirsAndFiles(fixSpec util.FixSpec) { + fixSpec.FixDir("var/zookeeper") + fixSpec.FixDir("var/zookeeper/conf") + fixSpec.FixDir("var/zookeeper/version-2") + fixSpec.FixFile("var/zookeeper/conf/zookeeper.cfg") + fixSpec.FixFile("var/zookeeper/myid") +} diff --git a/client/go/internal/admin/vespa-wrapper/configserver/logd.go b/client/go/internal/admin/vespa-wrapper/configserver/logd.go new file mode 100644 index 00000000000..39a2100b20f --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/configserver/logd.go @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func maybeStartLogd() { + v1 := os.Getenv(envvars.CLOUDCONFIG_SERVER_MULTITENANT) + v2 := os.Getenv(envvars.VESPA_CONFIGSERVER_MULTITENANT) + if v1 == "true" || v2 == "true" { + backticks := util.BackTicksForwardStderr + out, err := backticks.Run("libexec/vespa/start-logd") + if err != nil { + panic(err) + } + trace.Info(out) + } +} diff --git a/client/go/internal/admin/vespa-wrapper/configserver/runserver.go b/client/go/internal/admin/vespa-wrapper/configserver/runserver.go new file mode 100644 index 00000000000..bef50009d8d --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/configserver/runserver.go @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const ( + PROG_NAME = "vespa-runserver" +) + +type RunServer struct { + ServiceName string + Args []string +} + +func (rs *RunServer) PidFile() string { + varRunDir := defaults.UnderVespaHome("var/run") + return fmt.Sprintf("%s/%s.pid", varRunDir, rs.ServiceName) +} + +func (rs *RunServer) ProgPath() string { + p := fmt.Sprintf("%s/bin64/%s", defaults.VespaHome(), PROG_NAME) + if util.IsExecutableFile(p) { + return p + } + p = fmt.Sprintf("%s/bin/%s", defaults.VespaHome(), PROG_NAME) + if util.IsExecutableFile(p) { + return p + } + panic(fmt.Errorf("not an executable file: %s", p)) +} + +func (rs *RunServer) WouldRun() bool { + backticks := util.BackTicksForwardStderr + out, err := backticks.Run(rs.ProgPath(), "-s", rs.ServiceName, "-p", rs.PidFile(), "-W") + trace.Trace("output from -W:", out, "error:", err) + return err == nil +} + +func (rs *RunServer) Exec(prog string) { + argv := util.ArrayList[string]{ + PROG_NAME, + "-s", rs.ServiceName, + "-r", "30", + "-p", rs.PidFile(), + "--", + prog, + } + argv.AppendAll(rs.Args...) + err := util.Execvp(rs.ProgPath(), argv) + util.JustExitWith(err) +} diff --git a/client/go/internal/admin/vespa-wrapper/configserver/start.go b/client/go/internal/admin/vespa-wrapper/configserver/start.go new file mode 100644 index 00000000000..e2ea6a11357 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/configserver/start.go @@ -0,0 +1,86 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + SERVICE_NAME = "configserver" +) + +func commonPreChecks() (veHome string) { + if doTrace := os.Getenv(envvars.TRACE_JVM_STARTUP); doTrace != "" { + trace.AdjustVerbosity(1) + } + if doDebug := os.Getenv(envvars.DEBUG_JVM_STARTUP); doDebug != "" { + trace.AdjustVerbosity(2) + } + _ = vespa.FindAndVerifyVespaHome() + err := vespa.LoadDefaultEnv() + if err != nil { + panic(err) + } + veHome = vespa.FindAndVerifyVespaHome() + veHost, e := vespa.FindOurHostname() + if e != nil { + trace.Warning("could not detect hostname:", err, "; using fallback:", veHost) + } + checkIsConfigserver(veHost) + e = os.Chdir(veHome) + if e != nil { + util.JustExitWith(e) + } + return +} + +func JustStartConfigserver() int { + vespaHome := commonPreChecks() + vespa.CheckCorrectUser() + util.TuneResourceLimits() + util.TuneLogging(SERVICE_NAME, "com.google.api.client.http.HttpTransport", "config=off") + exportSettings(vespaHome) + removeStaleZkLocks(vespaHome) + c := jvm.NewStandaloneContainer(SERVICE_NAME) + jvmOpts := c.JvmOptions() + if extra := os.Getenv(envvars.VESPA_CONFIGSERVER_JVMARGS); extra != "" { + jvmOpts.AddJvmArgsFromString(extra) + } + minFallback := jvm.MegaBytesOfMemory(128) + maxFallback := jvm.MegaBytesOfMemory(2048) + jvmOpts.AddDefaultHeapSizeArgs(minFallback, maxFallback) + c.Exec() + // unreachable: + return 1 +} + +func runConfigserverWithRunserver() int { + commonPreChecks() + vespa.CheckCorrectUser() + rs := RunServer{ + ServiceName: SERVICE_NAME, + Args: []string{"just-start-configserver"}, + } + rs.Exec("libexec/vespa/vespa-wrapper") + return 1 +} + +func StartConfigserverEtc() int { + vespaHome := commonPreChecks() + vespa.RunPreStart() + util.TuneResourceLimits() + fixSpec := makeFixSpec() + fixDirsAndFiles(fixSpec) + exportSettings(vespaHome) + vespa.MaybeSwitchUser("vespa-start-configserver") + maybeStartLogd() + return runConfigserverWithRunserver() +} diff --git a/client/go/internal/admin/vespa-wrapper/configserver/zk.go b/client/go/internal/admin/vespa-wrapper/configserver/zk.go new file mode 100644 index 00000000000..91ddf522848 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/configserver/zk.go @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package configserver + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +const ( + ZOOKEEPER_LOG_FILE_PREFIX = "logs/vespa/zookeeper.configserver" +) + +func removeStaleZkLocks(vespaHome string) { + backticks := util.BackTicksIgnoreStderr + cmd := fmt.Sprintf("rm -f '%s/%s'*lck", vespaHome, ZOOKEEPER_LOG_FILE_PREFIX) + trace.Trace("cleaning locks:", cmd) + backticks.Run("/bin/sh", "-c", cmd) +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/cmd.go b/client/go/internal/admin/vespa-wrapper/logfmt/cmd.go new file mode 100644 index 00000000000..a8675c37356 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/cmd.go @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-logfmt command +// Author: arnej + +package logfmt + +import ( + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/cli/build" +) + +func NewLogfmtCmd() *cobra.Command { + var ( + curOptions Options = NewOptions() + ) + cmd := &cobra.Command{ + Use: "vespa-logfmt", + Short: "convert vespa.log to human-readable format", + Long: `vespa-logfmt takes input in the internal vespa format +and converts it to something human-readable`, + Version: build.Version, + Run: func(cmd *cobra.Command, args []string) { + RunLogfmt(&curOptions, args) + }, + } + cmd.Flags().VarP(&curOptions.ShowLevels, "level", "l", "turn levels on/off\n") + cmd.Flags().VarP(&curOptions.ShowFields, "show", "s", "turn fields shown on/off\n") + cmd.Flags().VarP(&curOptions.ComponentFilter, "component", "c", "select components by regexp") + cmd.Flags().VarP(&curOptions.MessageFilter, "message", "m", "select messages by regexp") + cmd.Flags().BoolVarP(&curOptions.OnlyInternal, "internal", "i", false, "select only internal components") + cmd.Flags().BoolVar(&curOptions.TruncateService, "truncateservice", false, "truncate service name") + cmd.Flags().BoolVar(&curOptions.TruncateService, "ts", false, "") + cmd.Flags().BoolVarP(&curOptions.FollowTail, "follow", "f", false, "follow logfile with tail -f") + cmd.Flags().BoolVarP(&curOptions.DequoteNewlines, "nldequote", "N", false, "dequote newlines embedded in message") + cmd.Flags().BoolVarP(&curOptions.DequoteNewlines, "dequotenewlines", "n", false, "dequote newlines embedded in message") + cmd.Flags().BoolVarP(&curOptions.TruncateComponent, "truncatecomponent", "t", false, "truncate component name") + cmd.Flags().BoolVar(&curOptions.TruncateComponent, "tc", false, "") + cmd.Flags().StringVarP(&curOptions.OnlyHostname, "host", "H", "", "select only one host") + cmd.Flags().StringVarP(&curOptions.OnlyPid, "pid", "p", "", "select only one process ID") + cmd.Flags().StringVarP(&curOptions.OnlyService, "service", "S", "", "select only one service") + cmd.Flags().VarP(&curOptions.Format, "format", "F", "select logfmt output format, vespa (default), json or raw are supported. The json output format is not stable, and will change in the future.") + cmd.Flags().MarkHidden("tc") + cmd.Flags().MarkHidden("ts") + cmd.Flags().MarkHidden("dequotenewlines") + return cmd +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/formatflags.go b/client/go/internal/admin/vespa-wrapper/logfmt/formatflags.go new file mode 100644 index 00000000000..097746d696f --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/formatflags.go @@ -0,0 +1,41 @@ +package logfmt + +import ( + "fmt" + "strings" +) + +type OutputFormat int + +const ( + FormatVespa OutputFormat = iota //default is vespa + FormatRaw + FormatJSON +) + +func (v *OutputFormat) Type() string { + return "output format" +} + +func (v *OutputFormat) String() string { + flagNames := []string{ + "vespa", + "raw", + "json", + } + return flagNames[*v] +} + +func (v *OutputFormat) Set(val string) error { + switch strings.ToLower(val) { + case "vespa": + *v = FormatVespa + case "raw": + *v = FormatRaw + case "json": + *v = FormatJSON + default: + return fmt.Errorf("'%s' is not a valid format argument", val) + } + return nil +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/formatflags_test.go b/client/go/internal/admin/vespa-wrapper/logfmt/formatflags_test.go new file mode 100644 index 00000000000..53c47d24208 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/formatflags_test.go @@ -0,0 +1,30 @@ +package logfmt + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestOutputFormat(t *testing.T) { + type args struct { + val string + } + tests := []struct { + expected OutputFormat + arg string + wantErr assert.ErrorAssertionFunc + }{ + {FormatVespa, "vespa", assert.NoError}, + {FormatRaw, "raw", assert.NoError}, + {FormatJSON, "json", assert.NoError}, + {-1, "foo", assert.Error}, + } + for _, tt := range tests { + t.Run(tt.arg, func(t *testing.T) { + var v OutputFormat = -1 + tt.wantErr(t, v.Set(tt.arg), fmt.Sprintf("Set(%v)", tt.arg)) + assert.Equal(t, v, tt.expected) + }) + } +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/handleline.go b/client/go/internal/admin/vespa-wrapper/logfmt/handleline.go new file mode 100644 index 00000000000..813ca82acb4 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/handleline.go @@ -0,0 +1,179 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" +) + +type logFields struct { + timestamp string // seconds, optional fractional seconds + host string + pid string // pid, optional tid + service string + component string + level string + messages []string +} + +// handle a line in "vespa.log" format; do filtering and formatting as specified in opts +func handleLine(opts *Options, line string) (string, error) { + fieldStrings := strings.SplitN(line, "\t", 7) + if len(fieldStrings) < 7 { + return "", fmt.Errorf("not enough fields: '%s'", line) + } + fields := logFields{ + timestamp: fieldStrings[0], + host: fieldStrings[1], + pid: fieldStrings[2], + service: fieldStrings[3], + component: fieldStrings[4], + level: fieldStrings[5], + messages: fieldStrings[6:], + } + + if !opts.showLevel(fields.level) { + return "", nil + } + if opts.OnlyHostname != "" && opts.OnlyHostname != fields.host { + return "", nil + } + if opts.OnlyPid != "" && opts.OnlyPid != fields.pid { + return "", nil + } + if opts.OnlyService != "" && opts.OnlyService != fields.service { + return "", nil + } + if opts.OnlyInternal && !isInternal(fields.component) { + return "", nil + } + if opts.ComponentFilter.unmatched(fields.component) { + return "", nil + } + if opts.MessageFilter.unmatched(strings.Join(fields.messages, "\t")) { + return "", nil + } + + switch opts.Format { + case FormatRaw: + return line + "\n", nil + case FormatJSON: + return handleLineJson(opts, &fields) + case FormatVespa: + fallthrough + default: + return handleLineVespa(opts, &fields) + } +} + +func parseTimestamp(timestamp string) (time.Time, error) { + secs, err := strconv.ParseFloat(timestamp, 64) + if err != nil { + return time.Time{}, err + } + nsecs := int64(secs * 1e9) + return time.Unix(0, nsecs), nil +} + +type logFieldsJson struct { + Timestamp string `json:"timestamp"` + Host string `json:"host"` + Pid string `json:"pid"` + Service string `json:"service"` + Component string `json:"component"` + Level string `json:"level"` + Messages []string `json:"messages"` +} + +func handleLineJson(_ *Options, fields *logFields) (string, error) { + timestamp, err := parseTimestamp(fields.timestamp) + if err != nil { + return "", err + } + outputFields := logFieldsJson{ + Timestamp: timestamp.Format(time.RFC3339Nano), + Host: fields.host, + Pid: fields.pid, + Service: fields.service, + Component: fields.component, + Level: fields.level, + Messages: fields.messages, + } + buf := bytes.Buffer{} + if err := json.NewEncoder(&buf).Encode(&outputFields); err != nil { + return "", err + } + return buf.String(), nil +} + +func handleLineVespa(opts *Options, fields *logFields) (string, error) { + var buf strings.Builder + + if opts.showField("fmttime") { + timestamp, err := parseTimestamp(fields.timestamp) + if err != nil { + return "", err + } + if opts.showField("usecs") { + buf.WriteString(timestamp.Format("[2006-01-02 15:04:05.000000] ")) + } else if opts.showField("msecs") { + buf.WriteString(timestamp.Format("[2006-01-02 15:04:05.000] ")) + } else { + buf.WriteString(timestamp.Format("[2006-01-02 15:04:05] ")) + } + } else if opts.showField("time") { + buf.WriteString(fields.timestamp) + buf.WriteString(" ") + } + if opts.showField("host") { + buf.WriteString(fmt.Sprintf("%-8s ", fields.host)) + } + if opts.showField("level") { + buf.WriteString(fmt.Sprintf("%-7s ", strings.ToUpper(fields.level))) + } + if opts.showField("pid") { + // OnlyPid, _, _ := strings.Cut(pidfield, "/") + buf.WriteString(fmt.Sprintf("%6s ", fields.pid)) + } + if opts.showField("service") { + if opts.TruncateService { + buf.WriteString(fmt.Sprintf("%-9.9s ", fields.service)) + } else { + buf.WriteString(fmt.Sprintf("%-16s ", fields.service)) + } + } + if opts.showField("component") { + if opts.TruncateComponent { + buf.WriteString(fmt.Sprintf("%-15.15s ", fields.component)) + } else { + buf.WriteString(fmt.Sprintf("%s\t", fields.component)) + } + } + if opts.showField("message") { + var msgBuf strings.Builder + for idx, message := range fields.messages { + if idx > 0 { + msgBuf.WriteString("\n\t") + } + if opts.DequoteNewlines { + message = strings.ReplaceAll(message, "\\n\\t", "\n\t") + message = strings.ReplaceAll(message, "\\n", "\n\t") + } + msgBuf.WriteString(message) + } + message := msgBuf.String() + if strings.Contains(message, "\n") { + buf.WriteString("\n\t") + } + buf.WriteString(message) + } + buf.WriteString("\n") + return buf.String(), nil +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/internal.go b/client/go/internal/admin/vespa-wrapper/logfmt/internal.go new file mode 100644 index 00000000000..992c537f939 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/internal.go @@ -0,0 +1,106 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" +) + +// is componentName a vespa-internal name? + +func isInternal(componentName string) bool { + cs := strings.Split(componentName, ".") + if len(cs) == 0 || cs[0] != "Container" { + return true + } + if len(cs) < 3 { + return false + } + if cs[1] == "ai" && cs[2] == "vespa" { + return true + } + if cs[1] == "com" && cs[2] == "yahoo" && len(cs) > 3 { + return internalComYahooNames[cs[3]] + } + return false +} + +// a constant: +var internalComYahooNames = map[string]bool{ + "abicheck": true, + "api": true, + "application": true, + "binaryprefix": true, + "clientmetrics": true, + "cloud": true, + "collections": true, + "component": true, + "compress": true, + "concurrent": true, + "configtest": true, + "config": true, + "container": true, + "data": true, + "docprocs": true, + "docproc": true, + "documentapi": true, + "documentmodel": true, + "document": true, + "dummyreceiver": true, + "embedding": true, + "errorhandling": true, + "exception": true, + "feedapi": true, + "feedhandler": true, + "filedistribution": true, + "fs4": true, + "fsa": true, + "geo": true, + "io": true, + "javacc": true, + "jdisc": true, + "jrt": true, + "lang": true, + "language": true, + "logserver": true, + "log": true, + "messagebus": true, + "metrics": true, + "nativec": true, + "net": true, + "osgi": true, + "path": true, + "plugin": true, + "prelude": true, + "processing": true, + "protect": true, + "reflection": true, + "restapi": true, + "schema": true, + "searchdefinition": true, + "searchlib": true, + "search": true, + "security": true, + "slime": true, + "socket": true, + "statistics": true, + "stream": true, + "system": true, + "tensor": true, + "test": true, + "text": true, + "time": true, + "transaction": true, + "vdslib": true, + "vespaclient": true, + "vespafeeder": true, + "vespaget": true, + "vespastat": true, + "vespasummarybenchmark": true, + "vespa": true, + "vespavisit": true, + "vespaxmlparser": true, + "yolean": true, +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/internal_names.txt b/client/go/internal/admin/vespa-wrapper/logfmt/internal_names.txt new file mode 100644 index 00000000000..cc554546fcc --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/internal_names.txt @@ -0,0 +1,782 @@ +sentinel.sentinel.config +searchnode.eval.Test +Container.ai.vespa.client.dsl.Test +Container.ai.vespa.cloud.Test +Container.ai.vespa.embedding.Test +Container.ai.vespa.explicitversion.Test +Container.ai.vespa.explicitversion_dep.Test +Container.ai.vespa.feed.client.Test +Container.ai.vespa.feed.client.impl.Test +Container.ai.vespa.hosted.api.Test +Container.ai.vespa.hosted.cd.Test +Container.ai.vespa.hosted.cd.cloud.impl.Test +Container.ai.vespa.hosted.cd.commons.Test +Container.ai.vespa.hosted.cd.internal.Test +Container.ai.vespa.hosted.client.Test +Container.ai.vespa.hosted.plugin.Test +Container.ai.vespa.http.Test +Container.ai.vespa.intellij.schema.Test +Container.ai.vespa.intellij.schema.findUsages.Test +Container.ai.vespa.intellij.schema.hierarchy.Test +Container.ai.vespa.intellij.schema.lexer.Test +Container.ai.vespa.intellij.schema.model.Test +Container.ai.vespa.intellij.schema.parser.Test +Container.ai.vespa.intellij.schema.psi.Test +Container.ai.vespa.intellij.schema.psi.impl.Test +Container.ai.vespa.intellij.schema.structure.Test +Container.ai.vespa.intellij.schema.utils.Test +Container.ai.vespa.logserver.protocol.Test +Container.ai.vespa.metricsproxy.core.Test +Container.ai.vespa.metricsproxy.http.Test +Container.ai.vespa.metricsproxy.http.application.Test +Container.ai.vespa.metricsproxy.http.metrics.Test +Container.ai.vespa.metricsproxy.http.prometheus.Test +Container.ai.vespa.metricsproxy.http.yamas.Test +Container.ai.vespa.metricsproxy.metric.Test +Container.ai.vespa.metricsproxy.metric.dimensions.Test +Container.ai.vespa.metricsproxy.metric.model.Test +Container.ai.vespa.metricsproxy.metric.model.json.Test +Container.ai.vespa.metricsproxy.metric.model.processing.Test +Container.ai.vespa.metricsproxy.metric.model.prometheus.Test +Container.ai.vespa.metricsproxy.node.Test +Container.ai.vespa.metricsproxy.rpc.Test +Container.ai.vespa.metricsproxy.service.Test +Container.ai.vespa.metricsproxy.telegraf.Test +Container.ai.vespa.modelintegration.evaluator.Test +Container.ai.vespa.models.evaluation.Test +Container.ai.vespa.models.handler.Test +Container.ai.vespa.noversion.Test +Container.ai.vespa.noversion_dep.Test +Container.ai.vespa.rankingexpression.importer.Test +Container.ai.vespa.rankingexpression.importer.configmodelview.Test +Container.ai.vespa.rankingexpression.importer.lightgbm.Test +Container.ai.vespa.rankingexpression.importer.onnx.Test +Container.ai.vespa.rankingexpression.importer.operations.Test +Container.ai.vespa.rankingexpression.importer.tensorflow.Test +Container.ai.vespa.rankingexpression.importer.vespa.Test +Container.ai.vespa.rankingexpression.importer.vespa.parser.Test +Container.ai.vespa.rankingexpression.importer.xgboost.Test +Container.ai.vespa.reindexing.Test +Container.ai.vespa.reindexing.http.Test +Container.ai.vespa.searchlib.searchprotocol.protobuf.Test +Container.ai.vespa.util.http.hc4.Test +Container.ai.vespa.util.http.hc4.retry.Test +Container.ai.vespa.util.http.hc5.Test +Container.ai.vespa.validation.Test +Container.com.yahoo.abicheck.classtree.Test +Container.com.yahoo.abicheck.collector.Test +Container.com.yahoo.abicheck.mojo.Test +Container.com.yahoo.abicheck.setmatcher.Test +Container.com.yahoo.abicheck.signature.Test +Container.com.yahoo.api.annotations.Test +Container.com.yahoo.application.Test +Container.com.yahoo.application.container.Test +Container.com.yahoo.application.container.handler.Test +Container.com.yahoo.application.container.impl.Test +Container.com.yahoo.application.content.Test +Container.com.yahoo.application.preprocessor.Test +Container.com.yahoo.binaryprefix.Test +Container.com.yahoo.clientmetrics.Test +Container.com.yahoo.cloud.config.Test +Container.com.yahoo.collections.Test +Container.com.yahoo.component.Test +Container.com.yahoo.component.annotation.Test +Container.com.yahoo.component.chain.Test +Container.com.yahoo.component.chain.dependencies.Test +Container.com.yahoo.component.chain.dependencies.ordering.Test +Container.com.yahoo.component.chain.model.Test +Container.com.yahoo.component.provider.Test +Container.com.yahoo.compress.Test +Container.com.yahoo.concurrent.Test +Container.com.yahoo.concurrent.classlock.Test +Container.com.yahoo.concurrent.maintenance.Test +Container.com.yahoo.config.Test +Container.com.yahoo.config.application.Test +Container.com.yahoo.config.application.api.Test +Container.com.yahoo.config.application.api.xml.Test +Container.com.yahoo.config.codegen.Test +Container.com.yahoo.config.docproc.Test +Container.com.yahoo.config.ini.Test +Container.com.yahoo.config.model.Test +Container.com.yahoo.config.model.admin.Test +Container.com.yahoo.config.model.api.Test +Container.com.yahoo.config.model.api.container.Test +Container.com.yahoo.config.model.application.Test +Container.com.yahoo.config.model.application.provider.Test +Container.com.yahoo.config.model.builder.xml.Test +Container.com.yahoo.config.model.deploy.Test +Container.com.yahoo.config.model.graph.Test +Container.com.yahoo.config.model.producer.Test +Container.com.yahoo.config.model.provision.Test +Container.com.yahoo.config.model.test.Test +Container.com.yahoo.config.provision.Test +Container.com.yahoo.config.provision.exception.Test +Container.com.yahoo.config.provision.host.Test +Container.com.yahoo.config.provisioning.Test +Container.com.yahoo.config.provision.security.Test +Container.com.yahoo.config.provision.serialization.Test +Container.com.yahoo.config.provision.zone.Test +Container.com.yahoo.config.subscription.Test +Container.com.yahoo.config.subscription.impl.Test +Container.com.yahoo.configtest.Test +Container.com.yahoo.config.text.Test +Container.com.yahoo.container.Test +Container.com.yahoo.container.bundle.Test +Container.com.yahoo.container.core.Test +Container.com.yahoo.container.core.config.Test +Container.com.yahoo.container.core.config.testutil.Test +Container.com.yahoo.container.core.document.Test +Container.com.yahoo.container.core.documentapi.Test +Container.com.yahoo.container.core.http.Test +Container.com.yahoo.container.core.identity.Test +Container.com.yahoo.container.di.Test +Container.com.yahoo.container.di.componentgraph.Test +Container.com.yahoo.container.di.componentgraph.core.Test +Container.com.yahoo.container.di.componentgraph.cycle.Test +Container.com.yahoo.container.di.config.Test +Container.com.yahoo.container.handler.Test +Container.com.yahoo.container.handler.metrics.Test +Container.com.yahoo.container.handler.observability.Test +Container.com.yahoo.container.handler.test.Test +Container.com.yahoo.container.handler.threadpool.Test +Container.com.yahoo.container.http.Test +Container.com.yahoo.container.http.filter.Test +Container.com.yahoo.container.jdisc.Test +Container.com.yahoo.container.jdisc.athenz.Test +Container.com.yahoo.container.jdisc.component.Test +Container.com.yahoo.container.jdisc.config.Test +Container.com.yahoo.container.jdisc.messagebus.Test +Container.com.yahoo.container.jdisc.metric.Test +Container.com.yahoo.container.jdisc.metric.state.Test +Container.com.yahoo.container.jdisc.secretstore.Test +Container.com.yahoo.container.jdisc.state.Test +Container.com.yahoo.container.jdisc.utils.Test +Container.com.yahoo.container.logging.Test +Container.com.yahoo.container.plugin.bundle.Test +Container.com.yahoo.container.plugin.classanalysis.Test +Container.com.yahoo.container.plugin.mojo.Test +Container.com.yahoo.container.plugin.osgi.Test +Container.com.yahoo.container.plugin.util.Test +Container.com.yahoo.container.protect.Test +Container.com.yahoo.container.standalone.Test +Container.com.yahoo.container.usability.Test +Container.com.yahoo.data.Test +Container.com.yahoo.data.access.Test +Container.com.yahoo.data.access.simple.Test +Container.com.yahoo.data.access.slime.Test +Container.com.yahoo.docproc.Test +Container.com.yahoo.docproc.impl.Test +Container.com.yahoo.docproc.jdisc.Test +Container.com.yahoo.docproc.jdisc.messagebus.Test +Container.com.yahoo.docproc.jdisc.metric.Test +Container.com.yahoo.docproc.jdisc.observability.Test +Container.com.yahoo.docproc.proxy.Test +Container.com.yahoo.docprocs.indexing.Test +Container.com.yahoo.document.Test +Container.com.yahoo.document.annotation.Test +Container.com.yahoo.documentapi.Test +Container.com.yahoo.documentapi.local.Test +Container.com.yahoo.documentapi.messagebus.Test +Container.com.yahoo.documentapi.messagebus.protocol.Test +Container.com.yahoo.documentapi.messagebus.systemstate.rule.Test +Container.com.yahoo.documentapi.metrics.Test +Container.com.yahoo.document.config.Test +Container.com.yahoo.document.datatypes.Test +Container.com.yahoo.document.fieldpathupdate.Test +Container.com.yahoo.document.fieldset.Test +Container.com.yahoo.document.idstring.Test +Container.com.yahoo.document.internal.Test +Container.com.yahoo.document.json.Test +Container.com.yahoo.document.json.document.Test +Container.com.yahoo.document.json.readers.Test +Container.com.yahoo.documentmodel.Test +Container.com.yahoo.document.predicate.Test +Container.com.yahoo.document.restapi.resource.Test +Container.com.yahoo.document.select.Test +Container.com.yahoo.document.select.convert.Test +Container.com.yahoo.document.select.parser.Test +Container.com.yahoo.document.select.rule.Test +Container.com.yahoo.document.select.simple.Test +Container.com.yahoo.document.serialization.Test +Container.com.yahoo.document.update.Test +Container.com.yahoo.dummyreceiver.Test +Container.com.yahoo.embedding.Test +Container.com.yahoo.errorhandling.Test +Container.com.yahoo.exception.Test +Container.com.yahoo.feedapi.Test +Container.com.yahoo.feedhandler.Test +Container.com.yahoo.filedistribution.fileacquirer.Test +Container.com.yahoo.fs4.Test +Container.com.yahoo.fsa.Test +Container.com.yahoo.fsa.conceptnet.Test +Container.com.yahoo.fsa.segmenter.Test +Container.com.yahoo.fsa.topicpredictor.Test +Container.com.yahoo.geo.Test +Container.com.yahoo.io.Test +Container.com.yahoo.io.reader.Test +Container.com.yahoo.javacc.Test +Container.com.yahoo.jdisc.Test +Container.com.yahoo.jdisc.application.Test +Container.com.yahoo.jdisc.bundle.Test +Container.com.yahoo.jdisc.bundle.a.Test +Container.com.yahoo.jdisc.bundle.b.Test +Container.com.yahoo.jdisc.bundle.c.Test +Container.com.yahoo.jdisc.bundle.d.Test +Container.com.yahoo.jdisc.bundle.e.Test +Container.com.yahoo.jdisc.bundle.f.Test +Container.com.yahoo.jdisc.bundle.g.Test +Container.com.yahoo.jdisc.bundle.g_act.Test +Container.com.yahoo.jdisc.bundle.h.Test +Container.com.yahoo.jdisc.bundle.i.Test +Container.com.yahoo.jdisc.bundle.j.Test +Container.com.yahoo.jdisc.bundle.k.Test +Container.com.yahoo.jdisc.bundle.l.Test +Container.com.yahoo.jdisc.bundle.m.Test +Container.com.yahoo.jdisc.bundle.my_act.Test +Container.com.yahoo.jdisc.bundle.n.Test +Container.com.yahoo.jdisc.bundle.o.Test +Container.com.yahoo.jdisc.bundle.p.Test +Container.com.yahoo.jdisc.bundle.q.Test +Container.com.yahoo.jdisc.bundle.r.Test +Container.com.yahoo.jdisc.bundle.s.Test +Container.com.yahoo.jdisc.bundle.t.Test +Container.com.yahoo.jdisc.bundle.u.Test +Container.com.yahoo.jdisc.client.Test +Container.com.yahoo.jdisc.cloud.aws.Test +Container.com.yahoo.jdisc.core.Test +Container.com.yahoo.jdisc.handler.Test +Container.com.yahoo.jdisc.http.Test +Container.com.yahoo.jdisc.http.cloud.Test +Container.com.yahoo.jdisc.http.filter.Test +Container.com.yahoo.jdisc.http.filter.chain.Test +Container.com.yahoo.jdisc.http.filter.security.athenz.Test +Container.com.yahoo.jdisc.http.filter.security.base.Test +Container.com.yahoo.jdisc.http.filter.security.cors.Test +Container.com.yahoo.jdisc.http.filter.security.csp.Test +Container.com.yahoo.jdisc.http.filter.security.misc.Test +Container.com.yahoo.jdisc.http.filter.security.rule.Test +Container.com.yahoo.jdisc.http.filter.util.Test +Container.com.yahoo.jdisc.http.server.jetty.Test +Container.com.yahoo.jdisc.http.server.jetty.testutils.Test +Container.com.yahoo.jdisc.http.ssl.impl.Test +Container.com.yahoo.jdisc.metrics.yamasconsumer.cloud.Test +Container.com.yahoo.jdisc.refcount.Test +Container.com.yahoo.jdisc.service.Test +Container.com.yahoo.jdisc.statistics.Test +Container.com.yahoo.jdisc.test.Test +Container.com.yahoo.jrt.Test +Container.com.yahoo.jrt.slobrok.Test +Container.com.yahoo.jrt.slobrok.api.Test +Container.com.yahoo.jrt.slobrok.server.Test +Container.com.yahoo.jrt.tool.Test +Container.com.yahoo.lang.Test +Container.com.yahoo.language.Test +Container.com.yahoo.language.detect.Test +Container.com.yahoo.language.opennlp.Test +Container.com.yahoo.language.process.Test +Container.com.yahoo.language.provider.Test +Container.com.yahoo.language.sentencepiece.Test +Container.com.yahoo.language.simple.Test +Container.com.yahoo.language.simple.kstem.Test +Container.com.yahoo.language.tools.Test +Container.com.yahoo.language.wordpiece.Test +Container.com.yahoo.log.Test +Container.com.yahoo.log.event.Test +Container.com.yahoo.log.impl.Test +Container.com.yahoo.logserver.Test +Container.com.yahoo.logserver.filter.Test +Container.com.yahoo.logserver.handlers.Test +Container.com.yahoo.logserver.handlers.archive.Test +Container.com.yahoo.logserver.handlers.logmetrics.Test +Container.com.yahoo.logserver.testutils.Test +Container.com.yahoo.messagebus.Test +Container.com.yahoo.messagebus.jdisc.Test +Container.com.yahoo.messagebus.jdisc.test.Test +Container.com.yahoo.messagebus.network.Test +Container.com.yahoo.messagebus.network.local.Test +Container.com.yahoo.messagebus.network.rpc.Test +Container.com.yahoo.messagebus.network.rpc.test.Test +Container.com.yahoo.messagebus.routing.Test +Container.com.yahoo.messagebus.routing.test.Test +Container.com.yahoo.messagebus.shared.Test +Container.com.yahoo.messagebus.test.Test +Container.com.yahoo.metrics.Test +Container.com.yahoo.metrics.simple.Test +Container.com.yahoo.metrics.simple.jdisc.Test +Container.com.yahoo.metrics.simple.runtime.Test +Container.com.yahoo.nativec.Test +Container.com.yahoo.net.Test +Container.com.yahoo.osgi.Test +Container.com.yahoo.osgi.annotation.Test +Container.com.yahoo.osgi.provider.model.Test +Container.com.yahoo.path.Test +Container.com.yahoo.plugin.Test +Container.com.yahoo.prelude.Test +Container.com.yahoo.prelude.cluster.Test +Container.com.yahoo.prelude.fastsearch.Test +Container.com.yahoo.prelude.hitfield.Test +Container.com.yahoo.prelude.query.Test +Container.com.yahoo.prelude.query.parser.Test +Container.com.yahoo.prelude.query.textualrepresentation.Test +Container.com.yahoo.prelude.querytransform.Test +Container.com.yahoo.prelude.searcher.Test +Container.com.yahoo.prelude.semantics.Test +Container.com.yahoo.prelude.semantics.benchmark.Test +Container.com.yahoo.prelude.semantics.config.Test +Container.com.yahoo.prelude.semantics.engine.Test +Container.com.yahoo.prelude.semantics.parser.Test +Container.com.yahoo.prelude.semantics.rule.Test +Container.com.yahoo.prelude.statistics.Test +Container.com.yahoo.processing.Test +Container.com.yahoo.processing.execution.Test +Container.com.yahoo.processing.execution.chain.Test +Container.com.yahoo.processing.handler.Test +Container.com.yahoo.processing.impl.Test +Container.com.yahoo.processing.processors.Test +Container.com.yahoo.processing.rendering.Test +Container.com.yahoo.processing.request.Test +Container.com.yahoo.processing.request.properties.Test +Container.com.yahoo.processing.response.Test +Container.com.yahoo.processing.test.Test +Container.com.yahoo.protect.Test +Container.com.yahoo.reflection.Test +Container.com.yahoo.restapi.Test +Container.com.yahoo.schema.Test +Container.com.yahoo.schema.derived.Test +Container.com.yahoo.schema.derived.validation.Test +Container.com.yahoo.schema.document.Test +Container.com.yahoo.schema.document.annotation.Test +Container.com.yahoo.schema.expressiontransforms.Test +Container.com.yahoo.schema.fieldoperation.Test +Container.com.yahoo.schema.parser.Test +Container.com.yahoo.schema.processing.Test +Container.com.yahoo.schema.processing.multifieldresolver.Test +Container.com.yahoo.search.Test +Container.com.yahoo.search.cluster.Test +Container.com.yahoo.search.config.Test +Container.com.yahoo.search.dispatch.Test +Container.com.yahoo.search.dispatch.rpc.Test +Container.com.yahoo.search.dispatch.searchcluster.Test +Container.com.yahoo.search.federation.Test +Container.com.yahoo.search.federation.selection.Test +Container.com.yahoo.search.federation.sourceref.Test +Container.com.yahoo.search.grouping.Test +Container.com.yahoo.search.grouping.request.Test +Container.com.yahoo.search.grouping.request.parser.Test +Container.com.yahoo.search.grouping.result.Test +Container.com.yahoo.search.grouping.vespa.Test +Container.com.yahoo.search.handler.Test +Container.com.yahoo.search.handler.observability.Test +Container.com.yahoo.search.intent.model.Test +Container.com.yahoo.searchlib.Test +Container.com.yahoo.searchlib.aggregation.Test +Container.com.yahoo.searchlib.aggregation.hll.Test +Container.com.yahoo.searchlib.document.Test +Container.com.yahoo.searchlib.expression.Test +Container.com.yahoo.searchlib.gbdt.Test +Container.com.yahoo.searchlib.rankingexpression.Test +Container.com.yahoo.searchlib.rankingexpression.evaluation.Test +Container.com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization.Test +Container.com.yahoo.searchlib.rankingexpression.evaluation.tensoroptimization.Test +Container.com.yahoo.searchlib.rankingexpression.parser.Test +Container.com.yahoo.searchlib.rankingexpression.rule.Test +Container.com.yahoo.searchlib.rankingexpression.transform.Test +Container.com.yahoo.searchlib.ranking.features.Test +Container.com.yahoo.searchlib.ranking.features.fieldmatch.Test +Container.com.yahoo.searchlib.tensor.Test +Container.com.yahoo.searchlib.treenet.Test +Container.com.yahoo.searchlib.treenet.parser.Test +Container.com.yahoo.searchlib.treenet.rule.Test +Container.com.yahoo.search.match.Test +Container.com.yahoo.search.pagetemplates.Test +Container.com.yahoo.search.pagetemplates.config.Test +Container.com.yahoo.search.pagetemplates.engine.Test +Container.com.yahoo.search.pagetemplates.engine.resolvers.Test +Container.com.yahoo.search.pagetemplates.model.Test +Container.com.yahoo.search.pagetemplates.result.Test +Container.com.yahoo.search.predicate.Test +Container.com.yahoo.search.predicate.annotator.Test +Container.com.yahoo.search.predicate.benchmarks.Test +Container.com.yahoo.search.predicate.index.Test +Container.com.yahoo.search.predicate.index.conjunction.Test +Container.com.yahoo.search.predicate.optimization.Test +Container.com.yahoo.search.predicate.serialization.Test +Container.com.yahoo.search.predicate.utils.Test +Container.com.yahoo.search.query.Test +Container.com.yahoo.search.query.context.Test +Container.com.yahoo.search.query.gui.Test +Container.com.yahoo.search.query.parser.Test +Container.com.yahoo.search.query.profile.Test +Container.com.yahoo.search.query.profile.compiled.Test +Container.com.yahoo.search.query.profile.config.Test +Container.com.yahoo.search.query.profile.types.Test +Container.com.yahoo.search.query.properties.Test +Container.com.yahoo.search.query.ranking.Test +Container.com.yahoo.search.query.restapi.Test +Container.com.yahoo.search.query.rewrite.Test +Container.com.yahoo.search.query.rewrite.rewriters.Test +Container.com.yahoo.search.query.textserialize.Test +Container.com.yahoo.search.query.textserialize.item.Test +Container.com.yahoo.search.query.textserialize.parser.Test +Container.com.yahoo.search.query.textserialize.serializer.Test +Container.com.yahoo.search.querytransform.Test +Container.com.yahoo.search.rendering.Test +Container.com.yahoo.search.result.Test +Container.com.yahoo.search.schema.Test +Container.com.yahoo.search.schema.internal.Test +Container.com.yahoo.search.searchchain.Test +Container.com.yahoo.search.searchchain.example.Test +Container.com.yahoo.search.searchchain.model.Test +Container.com.yahoo.search.searchchain.model.federation.Test +Container.com.yahoo.search.searchchain.testutil.Test +Container.com.yahoo.search.searchers.Test +Container.com.yahoo.search.statistics.Test +Container.com.yahoo.search.yql.Test +Container.com.yahoo.security.Test +Container.com.yahoo.security.tls.Test +Container.com.yahoo.slime.Test +Container.com.yahoo.socket.test.Test +Container.com.yahoo.stream.Test +Container.com.yahoo.system.Test +Container.com.yahoo.system.execution.Test +Container.com.yahoo.tensor.Test +Container.com.yahoo.tensor.evaluation.Test +Container.com.yahoo.tensor.functions.Test +Container.com.yahoo.tensor.serialization.Test +Container.com.yahoo.test.Test +Container.com.yahoo.test.json.Test +Container.com.yahoo.text.Test +Container.com.yahoo.text.internal.Test +Container.com.yahoo.text.interpretation.Test +Container.com.yahoo.time.Test +Container.com.yahoo.transaction.Test +Container.com.yahoo.vdslib.Test +Container.com.yahoo.vdslib.distribution.Test +Container.com.yahoo.vdslib.state.Test +Container.com.yahoo.vespa.Test +Container.com.yahoo.vespa.applicationmodel.Test +Container.com.yahoo.vespa.athenz.api.Test +Container.com.yahoo.vespa.athenz.aws.Test +Container.com.yahoo.vespa.athenz.client.Test +Container.com.yahoo.vespa.athenz.client.common.Test +Container.com.yahoo.vespa.athenz.client.common.bindings.Test +Container.com.yahoo.vespa.athenz.client.common.serializers.Test +Container.com.yahoo.vespa.athenz.client.zms.Test +Container.com.yahoo.vespa.athenz.client.zms.bindings.Test +Container.com.yahoo.vespa.athenz.client.zts.Test +Container.com.yahoo.vespa.athenz.client.zts.bindings.Test +Container.com.yahoo.vespa.athenz.client.zts.utils.Test +Container.com.yahoo.vespa.athenz.identity.Test +Container.com.yahoo.vespa.athenz.identityprovider.api.Test +Container.com.yahoo.vespa.athenz.identityprovider.api.bindings.Test +Container.com.yahoo.vespa.athenz.identityprovider.client.Test +Container.com.yahoo.vespa.athenz.tls.Test +Container.com.yahoo.vespa.athenz.utils.Test +Container.com.yahoo.vespa.athenz.zpe.Test +Container.com.yahoo.vespaclient.Test +Container.com.yahoo.vespa.clustercontroller.apps.clustercontroller.Test +Container.com.yahoo.vespa.clustercontroller.apputil.communication.http.Test +Container.com.yahoo.vespa.clustercontroller.core.Test +Container.com.yahoo.vespa.clustercontroller.core.database.Test +Container.com.yahoo.vespa.clustercontroller.core.hostinfo.Test +Container.com.yahoo.vespa.clustercontroller.core.listeners.Test +Container.com.yahoo.vespa.clustercontroller.core.restapiv2.Test +Container.com.yahoo.vespa.clustercontroller.core.restapiv2.requests.Test +Container.com.yahoo.vespa.clustercontroller.core.rpc.Test +Container.com.yahoo.vespa.clustercontroller.core.status.Test +Container.com.yahoo.vespa.clustercontroller.core.status.statuspage.Test +Container.com.yahoo.vespa.clustercontroller.utils.communication.async.Test +Container.com.yahoo.vespa.clustercontroller.utils.communication.http.Test +Container.com.yahoo.vespa.clustercontroller.utils.communication.http.writer.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.response.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.server.Test +Container.com.yahoo.vespa.clustercontroller.utils.util.Test +Container.com.yahoo.vespa.config.Test +Container.com.yahoo.vespa.config.benchmark.Test +Container.com.yahoo.vespa.config.buildergen.Test +Container.com.yahoo.vespa.config.content.Test +Container.com.yahoo.vespa.config.content.core.Test +Container.com.yahoo.vespa.config.content.reindexing.Test +Container.com.yahoo.vespa.config.core.Test +Container.com.yahoo.vespa.configdefinition.Test +Container.com.yahoo.vespa.configmodel.producers.Test +Container.com.yahoo.vespa.config.parser.Test +Container.com.yahoo.vespa.config.protocol.Test +Container.com.yahoo.vespa.config.proxy.Test +Container.com.yahoo.vespa.config.proxy.filedistribution.Test +Container.com.yahoo.vespa.config.search.Test +Container.com.yahoo.vespa.config.search.core.Test +Container.com.yahoo.vespa.config.server.Test +Container.com.yahoo.vespa.config.server.application.Test +Container.com.yahoo.vespa.config.server.configchange.Test +Container.com.yahoo.vespa.config.server.deploy.Test +Container.com.yahoo.vespa.config.server.filedistribution.Test +Container.com.yahoo.vespa.configserver.flags.Test +Container.com.yahoo.vespa.configserver.flags.db.Test +Container.com.yahoo.vespa.configserver.flags.http.Test +Container.com.yahoo.vespa.config.server.host.Test +Container.com.yahoo.vespa.config.server.http.Test +Container.com.yahoo.vespa.config.server.http.status.Test +Container.com.yahoo.vespa.config.server.http.v1.Test +Container.com.yahoo.vespa.config.server.http.v2.Test +Container.com.yahoo.vespa.config.server.http.v2.request.Test +Container.com.yahoo.vespa.config.server.http.v2.response.Test +Container.com.yahoo.vespa.config.server.maintenance.Test +Container.com.yahoo.vespa.config.server.metrics.Test +Container.com.yahoo.vespa.config.server.model.Test +Container.com.yahoo.vespa.config.server.modelfactory.Test +Container.com.yahoo.vespa.config.server.monitoring.Test +Container.com.yahoo.vespa.config.server.provision.Test +Container.com.yahoo.vespa.config.server.rpc.Test +Container.com.yahoo.vespa.config.server.rpc.security.Test +Container.com.yahoo.vespa.config.server.session.Test +Container.com.yahoo.vespa.config.server.tenant.Test +Container.com.yahoo.vespa.config.server.version.Test +Container.com.yahoo.vespa.config.server.zookeeper.Test +Container.com.yahoo.vespa.config.storage.Test +Container.com.yahoo.vespa.config.util.Test +Container.com.yahoo.vespa.curator.Test +Container.com.yahoo.vespa.curator.api.Test +Container.com.yahoo.vespa.curator.mock.Test +Container.com.yahoo.vespa.curator.recipes.Test +Container.com.yahoo.vespa.curator.stats.Test +Container.com.yahoo.vespa.curator.transaction.Test +Container.com.yahoo.vespa.defaults.Test +Container.com.yahoo.vespa.document.Test +Container.com.yahoo.vespa.document.dom.Test +Container.com.yahoo.vespa.documentmodel.Test +Container.com.yahoo.vespafeeder.Test +Container.com.yahoo.vespa.feed.perf.Test +Container.com.yahoo.vespa.filedistribution.Test +Container.com.yahoo.vespa.filedistribution.status.Test +Container.com.yahoo.vespa.flags.Test +Container.com.yahoo.vespa.flags.custom.Test +Container.com.yahoo.vespa.flags.file.Test +Container.com.yahoo.vespa.flags.json.Test +Container.com.yahoo.vespa.flags.json.wire.Test +Container.com.yahoo.vespaget.Test +Container.com.yahoo.vespa.hadoop.Test +Container.com.yahoo.vespa.hadoop.mapreduce.Test +Container.com.yahoo.vespa.hadoop.mapreduce.util.Test +Container.com.yahoo.vespa.hadoop.pig.Test +Container.com.yahoo.vespa.hosted.athenz.instanceproviderservice.Test +Container.com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.Test +Container.com.yahoo.vespa.hosted.ca.Test +Container.com.yahoo.vespa.hosted.ca.instance.Test +Container.com.yahoo.vespa.hosted.ca.restapi.Test +Container.com.yahoo.vespa.hosted.controller.Test +Container.com.yahoo.vespa.hosted.controller.api.application.v4.Test +Container.com.yahoo.vespa.hosted.controller.api.application.v4.model.Test +Container.com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.Test +Container.com.yahoo.vespa.hosted.controller.api.configserver.Test +Container.com.yahoo.vespa.hosted.controller.api.identifiers.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.archive.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.artifact.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.athenz.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.aws.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.billing.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.certificates.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.configserver.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.deployment.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.dns.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.entity.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.horizon.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.jira.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.maven.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.noderepository.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.organization.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.repair.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.resource.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.routing.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.secrets.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.stubs.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.user.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.vcmr.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.zone.Test +Container.com.yahoo.vespa.hosted.controller.api.role.Test +Container.com.yahoo.vespa.hosted.controller.api.systemflags.v1.Test +Container.com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.Test +Container.com.yahoo.vespa.hosted.controller.application.Test +Container.com.yahoo.vespa.hosted.controller.application.pkg.Test +Container.com.yahoo.vespa.hosted.controller.archive.Test +Container.com.yahoo.vespa.hosted.controller.athenz.Test +Container.com.yahoo.vespa.hosted.controller.athenz.config.Test +Container.com.yahoo.vespa.hosted.controller.athenz.impl.Test +Container.com.yahoo.vespa.hosted.controller.auditlog.Test +Container.com.yahoo.vespa.hosted.controller.certificate.Test +Container.com.yahoo.vespa.hosted.controller.concurrent.Test +Container.com.yahoo.vespa.hosted.controller.deployment.Test +Container.com.yahoo.vespa.hosted.controller.dns.Test +Container.com.yahoo.vespa.hosted.controller.maintenance.Test +Container.com.yahoo.vespa.hosted.controller.metric.Test +Container.com.yahoo.vespa.hosted.controller.notification.Test +Container.com.yahoo.vespa.hosted.controller.persistence.Test +Container.com.yahoo.vespa.hosted.controller.proxy.Test +Container.com.yahoo.vespa.hosted.controller.restapi.application.Test +Container.com.yahoo.vespa.hosted.controller.restapi.athenz.Test +Container.com.yahoo.vespa.hosted.controller.restapi.billing.Test +Container.com.yahoo.vespa.hosted.controller.restapi.changemanagement.Test +Container.com.yahoo.vespa.hosted.controller.restapi.configserver.Test +Container.com.yahoo.vespa.hosted.controller.restapi.controller.Test +Container.com.yahoo.vespa.hosted.controller.restapi.deployment.Test +Container.com.yahoo.vespa.hosted.controller.restapi.filter.Test +Container.com.yahoo.vespa.hosted.controller.restapi.flags.Test +Container.com.yahoo.vespa.hosted.controller.restapi.horizon.Test +Container.com.yahoo.vespa.hosted.controller.restapi.os.Test +Container.com.yahoo.vespa.hosted.controller.restapi.routing.Test +Container.com.yahoo.vespa.hosted.controller.restapi.systemflags.Test +Container.com.yahoo.vespa.hosted.controller.restapi.user.Test +Container.com.yahoo.vespa.hosted.controller.restapi.zone.v1.Test +Container.com.yahoo.vespa.hosted.controller.restapi.zone.v2.Test +Container.com.yahoo.vespa.hosted.controller.routing.Test +Container.com.yahoo.vespa.hosted.controller.routing.context.Test +Container.com.yahoo.vespa.hosted.controller.routing.rotation.Test +Container.com.yahoo.vespa.hosted.controller.security.Test +Container.com.yahoo.vespa.hosted.controller.support.access.Test +Container.com.yahoo.vespa.hosted.controller.tenant.Test +Container.com.yahoo.vespa.hosted.controller.tls.Test +Container.com.yahoo.vespa.hosted.controller.versions.Test +Container.com.yahoo.vespa.hosted.node.admin.Test +Container.com.yahoo.vespa.hosted.node.admin.component.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.flags.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.state.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.Test +Container.com.yahoo.vespa.hosted.node.admin.container.Test +Container.com.yahoo.vespa.hosted.node.admin.container.image.Test +Container.com.yahoo.vespa.hosted.node.admin.container.metrics.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.acl.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.coredump.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.disk.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.identity.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.sync.Test +Container.com.yahoo.vespa.hosted.node.admin.nodeadmin.Test +Container.com.yahoo.vespa.hosted.node.admin.nodeagent.Test +Container.com.yahoo.vespa.hosted.node.admin.provider.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.editor.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.file.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.fs.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.network.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.process.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.systemd.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.template.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.text.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.yum.Test +Container.com.yahoo.vespa.hosted.provision.Test +Container.com.yahoo.vespa.hosted.provision.applications.Test +Container.com.yahoo.vespa.hosted.provision.autoscale.Test +Container.com.yahoo.vespa.hosted.provision.lb.Test +Container.com.yahoo.vespa.hosted.provision.maintenance.Test +Container.com.yahoo.vespa.hosted.provision.node.Test +Container.com.yahoo.vespa.hosted.provision.node.filter.Test +Container.com.yahoo.vespa.hosted.provision.os.Test +Container.com.yahoo.vespa.hosted.provision.persistence.Test +Container.com.yahoo.vespa.hosted.provision.provisioning.Test +Container.com.yahoo.vespa.hosted.provision.restapi.Test +Container.com.yahoo.vespa.hosted.provision.testutils.Test +Container.com.yahoo.vespa.hosted.routing.Test +Container.com.yahoo.vespa.hosted.routing.nginx.Test +Container.com.yahoo.vespa.hosted.routing.restapi.Test +Container.com.yahoo.vespa.hosted.routing.status.Test +Container.com.yahoo.vespa.hosted.testrunner.Test +Container.com.yahoo.vespa.http.server.Test +Container.com.yahoo.vespa.http.server.util.Test +Container.com.yahoo.vespa.indexinglanguage.Test +Container.com.yahoo.vespa.indexinglanguage.expressions.Test +Container.com.yahoo.vespa.indexinglanguage.linguistics.Test +Container.com.yahoo.vespa.indexinglanguage.parser.Test +Container.com.yahoo.vespa.indexinglanguage.predicate.Test +Container.com.yahoo.vespa.jaxrs.annotation.Test +Container.com.yahoo.vespa.maven.plugin.enforcer.Test +Container.com.yahoo.vespa.model.Test +Container.com.yahoo.vespa.model.admin.Test +Container.com.yahoo.vespa.model.admin.clustercontroller.Test +Container.com.yahoo.vespa.model.admin.metricsproxy.Test +Container.com.yahoo.vespa.model.admin.monitoring.Test +Container.com.yahoo.vespa.model.admin.monitoring.builder.Test +Container.com.yahoo.vespa.model.admin.monitoring.builder.xml.Test +Container.com.yahoo.vespa.model.application.validation.Test +Container.com.yahoo.vespa.model.application.validation.change.Test +Container.com.yahoo.vespa.model.application.validation.change.search.Test +Container.com.yahoo.vespa.model.application.validation.first.Test +Container.com.yahoo.vespa.model.builder.Test +Container.com.yahoo.vespa.model.builder.xml.dom.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.docproc.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.processing.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.search.Test +Container.com.yahoo.vespa.model.clients.Test +Container.com.yahoo.vespa.model.container.Test +Container.com.yahoo.vespa.model.container.component.Test +Container.com.yahoo.vespa.model.container.component.chain.Test +Container.com.yahoo.vespa.model.container.configserver.Test +Container.com.yahoo.vespa.model.container.configserver.option.Test +Container.com.yahoo.vespa.model.container.docproc.Test +Container.com.yahoo.vespa.model.container.docproc.model.Test +Container.com.yahoo.vespa.model.container.http.Test +Container.com.yahoo.vespa.model.container.http.ssl.Test +Container.com.yahoo.vespa.model.container.http.xml.Test +Container.com.yahoo.vespa.model.container.ml.Test +Container.com.yahoo.vespa.model.container.processing.Test +Container.com.yahoo.vespa.model.container.search.Test +Container.com.yahoo.vespa.model.container.search.searchchain.Test +Container.com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains.Test +Container.com.yahoo.vespa.model.container.xml.Test +Container.com.yahoo.vespa.model.container.xml.document.Test +Container.com.yahoo.vespa.model.container.xml.embedder.Test +Container.com.yahoo.vespa.model.content.Test +Container.com.yahoo.vespa.model.content.cluster.Test +Container.com.yahoo.vespa.model.content.engines.Test +Container.com.yahoo.vespa.model.content.storagecluster.Test +Container.com.yahoo.vespa.model.filedistribution.Test +Container.com.yahoo.vespa.model.ml.Test +Container.com.yahoo.vespa.model.routing.Test +Container.com.yahoo.vespa.model.search.Test +Container.com.yahoo.vespa.model.utils.Test +Container.com.yahoo.vespa.model.utils.internal.Test +Container.com.yahoo.vespa.objects.Test +Container.com.yahoo.vespa.orchestrator.Test +Container.com.yahoo.vespa.orchestrator.config.Test +Container.com.yahoo.vespa.orchestrator.controller.Test +Container.com.yahoo.vespa.orchestrator.model.Test +Container.com.yahoo.vespa.orchestrator.policy.Test +Container.com.yahoo.vespa.orchestrator.resources.Test +Container.com.yahoo.vespa.orchestrator.restapi.wire.Test +Container.com.yahoo.vespa.orchestrator.status.Test +Container.com.yahoo.vespa.orchestrator.status.json.Test +Container.com.yahoo.vespa.security.tool.securityenv.Test +Container.com.yahoo.vespa.service.duper.Test +Container.com.yahoo.vespa.service.executor.Test +Container.com.yahoo.vespa.service.health.Test +Container.com.yahoo.vespa.service.manager.Test +Container.com.yahoo.vespa.service.model.Test +Container.com.yahoo.vespa.service.monitor.Test +Container.com.yahoo.vespa.service.slobrok.Test +Container.com.yahoo.vespastat.Test +Container.com.yahoo.vespa.streamingvisitors.Test +Container.com.yahoo.vespa.streamingvisitors.tracing.Test +Container.com.yahoo.vespasummarybenchmark.Test +Container.com.yahoo.vespa.test.file.Test +Container.com.yahoo.vespa.testrunner.Test +Container.com.yahoo.vespavisit.Test +Container.com.yahoo.vespaxmlparser.Test +Container.com.yahoo.vespa.zookeeper.Test +Container.com.yahoo.vespa.zookeeper.cli.Test +Container.com.yahoo.vespa.zookeeper.client.Test +Container.com.yahoo.yolean.Test +Container.com.yahoo.yolean.chain.Test +Container.com.yahoo.yolean.concurrent.Test +Container.com.yahoo.yolean.function.Test +Container.com.yahoo.yolean.system.Test +Container.com.yahoo.yolean.trace.Test diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/internal_notnames.txt b/client/go/internal/admin/vespa-wrapper/logfmt/internal_notnames.txt new file mode 100644 index 00000000000..49543758ac3 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/internal_notnames.txt @@ -0,0 +1,8 @@ +Container.ai.onnx.Test +Container.com.fasterxml.jackson.jaxrs.json.Test +Container.com.google.Test +Container.com.yahooapis.foo.Test +Container.com.yahoo.newssearch.Test +Container.com.yahoo.Test +Container.com.yahoo.testing.Test +Container.org.apache.http.Test diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/internal_test.go b/client/go/internal/admin/vespa-wrapper/logfmt/internal_test.go new file mode 100644 index 00000000000..9b6b0f8404c --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/internal_test.go @@ -0,0 +1,34 @@ +package logfmt + +import ( + "bufio" + "os" + "testing" +) + +// tests: func isInternal(componentName string) bool + +func TestIsInternal(t *testing.T) { + f, err := os.Open("internal_names.txt") + if err != nil { + t.Fatal("could not read test data") + } + defer f.Close() + for input := bufio.NewScanner(f); input.Scan(); { + if name := input.Text(); !isInternal(name) { + t.Logf("name '%s' should be internal but was not recognized", name) + t.Fail() + } + } + f, err = os.Open("internal_notnames.txt") + if err != nil { + t.Fatal("could not read test data") + } + defer f.Close() + for input := bufio.NewScanner(f); input.Scan(); { + if name := input.Text(); isInternal(name) { + t.Logf("name '%s' should not be internal but was recognized", name) + t.Fail() + } + } +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/levelflags.go b/client/go/internal/admin/vespa-wrapper/logfmt/levelflags.go new file mode 100644 index 00000000000..4e6c1284753 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/levelflags.go @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" +) + +// handle CLI flags for log level filtering + +type flagValueForLevel struct { + levels map[string]bool + changed bool +} + +func defaultLevelFlags() map[string]bool { + return map[string]bool{ + "fatal": true, + "error": true, + "warning": true, + "info": true, + "config": false, + "event": false, + "debug": false, + "spam": false, + } +} + +func (v *flagValueForLevel) Type() string { + return "level flags" +} + +func (v *flagValueForLevel) String() string { + var buf strings.Builder + flagNames := []string{ + "fatal", + "error", + "warning", + "info", + "config", + "event", + "debug", + "spam", + } + for _, flag := range flagNames { + if v.levels[flag] { + buf.WriteString(" +") + } else { + buf.WriteString(" -") + } + buf.WriteString(flag) + } + return buf.String() +} + +func (v *flagValueForLevel) flags() map[string]bool { + return v.levels +} + +func (v *flagValueForLevel) name() string { + return "level" +} + +func (v *flagValueForLevel) unchanged() bool { + return !v.changed +} + +func (v *flagValueForLevel) Set(val string) error { + return applyPlusMinus(val, v) +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/levelflags_test.go b/client/go/internal/admin/vespa-wrapper/logfmt/levelflags_test.go new file mode 100644 index 00000000000..186ea2d96b0 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/levelflags_test.go @@ -0,0 +1,74 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" + "testing" +) + +func TestLevelFlags(t *testing.T) { + none := " -fatal -error -warning -info -config -event -debug -spam" + + var flag flagValueForLevel + if flag.String() != none { + t.Logf("unset flag displays as '%s', expected '%s'", flag.String(), none) + t.Fail() + } + if flag.Type() != "level flags" { + t.Logf("flag type was '%s'", flag.Type()) + t.Fail() + } + check := func(expected string, texts ...string) { + var target flagValueForLevel + // target.levels = defaultLevelFlags() + target.levels = defaultLevelFlags() + for _, text := range texts { + err := target.Set(text) + if err != nil { + t.Fatalf("unexpected error with level flags Set('%s'): %v", text, err) + } + } + got := target.String() + if got != expected { + t.Logf("expected flags [%s] but got: [%s]", expected, got) + t.Fail() + } + } + check(" +fatal +error +warning +info -config -event -debug -spam") + check(" -fatal -error -warning -info -config -event -debug -spam", "-all") + check(" +fatal +error +warning +info +config +event +debug +spam", "all") + check(" +fatal +error +warning +info +config +event +debug +spam", "+all") + check(" -fatal -error -warning -info -config -event +debug -spam", "debug") + check(" +fatal +error +warning +info +config +event +debug -spam", "all-spam") + check(" +fatal +error +warning +info +config +event +debug -spam", "all", "-spam") + check(" +fatal +error +warning -info -config +event -debug -spam", "+event", "-info") + check(" +fatal +error -warning -info -config +event -debug -spam", "+event,-info,-warning,config") + check(" +fatal +error -warning -info +config +event -debug -spam", "+event,-info,-warning,+config") + check(" +fatal +error -warning -info +config +event -debug -spam", "+event,-info", "-warning,+config") + check(" -fatal -error -warning -info +config -event -debug -spam", "+event", "-info", "-warning", "config") + check = func(expectErr string, texts ...string) { + var target flagValueForLevel + target.levels = defaultLevelFlags() + for _, text := range texts { + err := target.Set(text) + if err != nil { + if err.Error() == expectErr { + return + } + t.Fatalf("expected error [%s] with level flags Set('%s'), but got [%v]", expectErr, text, err) + } + } + t.Logf("Did not get expected error '%s' from %s", expectErr, strings.Join(texts, ",")) + t.Fail() + } + check("not a valid level flag: 'foo'", "foo") + check("not a valid level flag: 'foo'", "event,foo,config") + check("not a valid level flag: 'foo'", "-event,-foo,-config") + check("not a valid level flag: 'foo'", "+event,+foo,+config") + check("not a valid level flag: 'foo'", "event", "foo", "config") + check("not a valid level flag: 'foo'", "-event", "-foo", "-config") + check("not a valid level flag: 'foo'", "+event", "+foo", "+config") +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/options.go b/client/go/internal/admin/vespa-wrapper/logfmt/options.go new file mode 100644 index 00000000000..864868d4ce5 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/options.go @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "fmt" + "os" +) + +// options designed for compatibility with perl version of vespa-logfmt + +type Options struct { + ShowFields flagValueForShow + ShowLevels flagValueForLevel + OnlyHostname string + OnlyPid string + OnlyService string + OnlyInternal bool + FollowTail bool + DequoteNewlines bool + TruncateService bool + TruncateComponent bool + ComponentFilter regexFlag + MessageFilter regexFlag + Format OutputFormat +} + +func NewOptions() (ret Options) { + ret.ShowLevels.levels = defaultLevelFlags() + ret.ShowFields.shown = defaultShowFlags() + return +} + +func (o *Options) showField(field string) bool { + return o.ShowFields.shown[field] +} + +func (o *Options) showLevel(level string) bool { + rv, ok := o.ShowLevels.levels[level] + if !ok { + o.ShowLevels.levels[level] = true + fmt.Fprintf(os.Stderr, "Warnings: unknown level '%s' in input\n", level) + return true + } + return rv +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/plusminusflag.go b/client/go/internal/admin/vespa-wrapper/logfmt/plusminusflag.go new file mode 100644 index 00000000000..1768cf0e7be --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/plusminusflag.go @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "fmt" + "strings" +) + +// common code for showFlags and levelFlags + +type plusMinusFlag interface { + flags() map[string]bool + name() string + unchanged() bool +} + +func trimPrefix(value, prefix string) (newValue string, hadPrefix bool) { + hadPrefix = strings.HasPrefix(value, prefix) + if hadPrefix { + newValue = strings.TrimPrefix(value, prefix) + } else { + newValue = value + } + return +} + +func applyPlusMinus(val string, target plusMinusFlag) error { + minus := strings.HasPrefix(val, "-") + plus := strings.HasPrefix(val, "+") + val = strings.ReplaceAll(val, "-", ",-") + val = strings.ReplaceAll(val, "+", ",+") + if target.unchanged() { + // user wants to reset flags? + if minus == false && plus == false { + for k, _ := range target.flags() { + target.flags()[k] = false + } + } + } + changeTo := !minus + for _, k := range strings.Split(val, ",") { + if suppress, minus := trimPrefix(k, "-"); minus { + k = suppress + changeTo = false + } + if surface, plus := trimPrefix(k, "+"); plus { + k = surface + changeTo = true + } + if k == "" { + continue + } + if k == "all" { + for k, _ := range target.flags() { + target.flags()[k] = changeTo + } + } else if _, ok := target.flags()[k]; !ok { + return fmt.Errorf("not a valid %s flag: '%s'", target.name(), k) + } else { + target.flags()[k] = changeTo + } + } + return nil +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/regexflag.go b/client/go/internal/admin/vespa-wrapper/logfmt/regexflag.go new file mode 100644 index 00000000000..8f7d2a91373 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/regexflag.go @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "regexp" +) + +// optional regular expression filter, as a CLI flag + +type regexFlag struct { + regex *regexp.Regexp +} + +func (re regexFlag) unmatched(s string) bool { + if re.regex == nil { + return false + } + return re.regex.FindStringIndex(s) == nil +} + +func (v *regexFlag) Type() string { + return "regular expression" +} + +func (v *regexFlag) String() string { + if v.regex == nil { + return "" + } + return v.regex.String() +} + +func (v *regexFlag) Set(val string) (r error) { + v.regex, r = regexp.Compile(val) + return +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/regexflag_test.go b/client/go/internal/admin/vespa-wrapper/logfmt/regexflag_test.go new file mode 100644 index 00000000000..489439863a2 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/regexflag_test.go @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func assertMatch(t *testing.T, re string, flag *regexFlag, texts ...string) { + t.Helper() + err := flag.Set(re) + require.Nil(t, err, "unexpected error with flag.Set('%s'): %v", re, err) + assert.Equal(t, re, flag.String(), "set flag displays as '%s', expected '%s'", flag.String(), re) + for _, text := range texts { + assert.False(t, flag.unmatched(text), "flag '%s' claims a non-match for '%s'", flag.String(), text) + } +} + +func assertUnmatch(t *testing.T, re string, flag *regexFlag, texts ...string) { + t.Helper() + err := flag.Set(re) + require.Nil(t, err, "unexpected error with flag.Set('%s'): %v", re, err) + assert.Equal(t, re, flag.String()) + for _, text := range texts { + assert.True(t, flag.unmatched(text), "flag '%s' should claim a non-match for '%s'", flag.String(), text) + } +} + +func TestRegexFlag(t *testing.T) { + var flag regexFlag + assert.Equal(t, "", flag.String()) + assert.Equal(t, "regular expression", flag.Type()) + assert.False(t, flag.unmatched("foobar"), "unset flag claims a non-match") + assert.EqualError(t, flag.Set("*"), "error parsing regexp: missing argument to repetition operator: `*`") + assertMatch(t, "foo.*bar", new(regexFlag), "foobar", "foo bar", "x foobar y", "xfoobary", "xfooybarz") + assertUnmatch(t, "foo.*bar", new(regexFlag), "Foobar", "foo Bar", "fxoobar", "whatever") +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/runlogfmt.go b/client/go/internal/admin/vespa-wrapper/logfmt/runlogfmt.go new file mode 100644 index 00000000000..6557461598e --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/runlogfmt.go @@ -0,0 +1,85 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "bufio" + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func inputIsPipe() bool { + fi, err := os.Stdin.Stat() + if err != nil { + return false + } + if fi.Mode()&os.ModeNamedPipe == 0 { + return false + } else { + return true + } +} + +// main entry point for vespa-logfmt + +func RunLogfmt(opts *Options, args []string) { + if len(args) == 0 { + if !inputIsPipe() { + args = append(args, vespa.FindHome()+"/logs/vespa/vespa.log") + } else { + formatFile(opts, os.Stdin) + } + } + if opts.FollowTail { + if len(args) != 1 { + fmt.Fprintf(os.Stderr, "Must have exact 1 file for 'follow' option, got %d\n", len(args)) + return + } + if err := tailFile(opts, args[0]); err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + return + } + for _, arg := range args { + file, err := os.Open(arg) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot open '%s': %v\n", arg, err) + } else { + formatFile(opts, file) + file.Close() + } + } +} + +func formatLine(opts *Options, line string) { + output, err := handleLine(opts, line) + if err != nil { + fmt.Fprintln(os.Stderr, "bad log line:", err) + } else { + os.Stdout.WriteString(output) + } +} + +func tailFile(opts *Options, fn string) error { + tailed, err := FollowFile(fn) + if err != nil { + return err + } + for line := range tailed.Lines() { + formatLine(opts, line.Text) + } + return nil +} + +func formatFile(opts *Options, arg *os.File) { + input := bufio.NewScanner(arg) + input.Buffer(make([]byte, 64*1024), 4*1024*1024) + for input.Scan() { + formatLine(opts, input.Text()) + } +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/showflags.go b/client/go/internal/admin/vespa-wrapper/logfmt/showflags.go new file mode 100644 index 00000000000..b69860e0312 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/showflags.go @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" +) + +// handle CLI flags for which fields to show when formatting a line + +type flagValueForShow struct { + shown map[string]bool + changed bool +} + +func defaultShowFlags() map[string]bool { + return map[string]bool{ + "time": true, + "fmttime": true, + "msecs": true, + "usecs": false, + "host": false, + "level": true, + "pid": false, + "service": true, + "component": true, + "message": true, + } +} + +func (v *flagValueForShow) Type() string { + return "show flags" +} + +func (v *flagValueForShow) String() string { + var buf strings.Builder + flagNames := []string{ + "time", + "fmttime", + "msecs", + "usecs", + "host", + "level", + "pid", + "service", + "component", + "message", + } + for _, flag := range flagNames { + if v.shown[flag] { + buf.WriteString(" +") + } else { + buf.WriteString(" -") + } + buf.WriteString(flag) + } + return buf.String() +} + +func (v *flagValueForShow) flags() map[string]bool { + return v.shown +} + +func (v *flagValueForShow) name() string { + return "show" +} + +func (v *flagValueForShow) unchanged() bool { + return !v.changed +} + +func (v *flagValueForShow) Set(val string) error { + return applyPlusMinus(val, v) +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/showflags_test.go b/client/go/internal/admin/vespa-wrapper/logfmt/showflags_test.go new file mode 100644 index 00000000000..d1b66118afd --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/showflags_test.go @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "strings" + "testing" +) + +func TestShowFlags(t *testing.T) { + none := " -time -fmttime -msecs -usecs -host -level -pid -service -component -message" + var flag flagValueForShow + assert.Equal(t, none, flag.String(), "unset flag displays as '%s', expected '%s'", flag.String(), none) + assert.Equal(t, "show flags", flag.Type()) + check := func(expected string, texts ...string) { + var target flagValueForShow + // target.levels = defaultLevelFlags() + target.shown = defaultShowFlags() + for _, text := range texts { + err := target.Set(text) + require.Nil(t, err, "unexpected error with show flags Set('%s'): %v", text, err) + } + assert.Equal(t, expected, target.String()) + } + check(" +time +fmttime +msecs -usecs -host +level -pid +service +component +message") + check(" -time -fmttime -msecs -usecs -host -level -pid -service -component -message", "-all") + check(" +time +fmttime +msecs +usecs +host +level +pid +service +component +message", "all") + check(" +time +fmttime +msecs +usecs +host +level +pid +service +component +message", "+all") + check(" -time -fmttime -msecs -usecs -host -level +pid -service -component -message", "pid") + check(" +time +fmttime +msecs +usecs -host +level +pid +service +component +message", "all-host") + check(" +time +fmttime +msecs +usecs -host +level +pid +service +component +message", "all", "-host") + check(" +time +fmttime -msecs -usecs -host +level +pid +service +component +message", "+pid", "-msecs") + check(" +time -fmttime +msecs -usecs +host +level -pid -service +component +message", "+host,-fmttime,-service,pid") + check(" +time -fmttime +msecs -usecs +host +level +pid -service +component +message", "+host,-fmttime,-service,+pid") + check(" +time -fmttime +msecs -usecs +host +level +pid -service +component +message", "+host,-fmttime", "-service,+pid") + check(" -time -fmttime -msecs -usecs -host -level +pid -service -component -message", "+host", "-fmttime", "-service", "pid") + check = func(expectErr string, texts ...string) { + var target flagValueForShow + target.shown = defaultShowFlags() + for _, text := range texts { + err := target.Set(text) + if err != nil { + require.Equal(t, expectErr, err.Error()) + return + } + } + t.Logf("Did not get expected error [%s] from %s", expectErr, strings.Join(texts, " ")) + t.Fail() + } + check("not a valid show flag: 'foo'", "foo") + check("not a valid show flag: 'foo'", "level,foo,message") + check("not a valid show flag: 'foo'", "-level,-foo,-message") + check("not a valid show flag: 'foo'", "+level,+foo,+message") + check("not a valid show flag: 'foo'", "level", "foo", "message") + check("not a valid show flag: 'foo'", "-level", "-foo", "-message") + check("not a valid show flag: 'foo'", "+level", "+foo", "+message") +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/tail.go b/client/go/internal/admin/vespa-wrapper/logfmt/tail.go new file mode 100644 index 00000000000..75e7cbb0693 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/tail.go @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: mpolden + +package logfmt + +type Line struct { + Text string +} + +type Tail interface { + Lines() chan Line +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/tail_not_unix.go b/client/go/internal/admin/vespa-wrapper/logfmt/tail_not_unix.go new file mode 100644 index 00000000000..7030572575d --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/tail_not_unix.go @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: mpolden + +//go:build windows + +package logfmt + +import ( + "fmt" +) + +func FollowFile(fn string) (Tail, error) { + return nil, fmt.Errorf("tail is not supported on this platform") +} diff --git a/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go b/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go new file mode 100644 index 00000000000..7703844da48 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/logfmt/tail_unix.go @@ -0,0 +1,170 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +//go:build !windows + +package logfmt + +import ( + "bufio" + "fmt" + "io" + "os" + "time" + + "golang.org/x/sys/unix" +) + +const lastLinesSize = 4 * 1024 + +// an active "tail -f" like object + +type unixTail struct { + lines chan Line + lineBuf []byte + curFile *os.File + fn string + reader *bufio.Reader + curStat unix.Stat_t +} + +func (t *unixTail) Lines() chan Line { return t.lines } + +// API for starting to follow a log file + +func FollowFile(fn string) (Tail, error) { + res := unixTail{} + res.fn = fn + res.lineBuf = make([]byte, lastLinesSize) + res.openTail() + res.lines = make(chan Line, 20) + res.lineBuf = res.lineBuf[:0] + go runTailWith(&res) + return &res, nil +} + +func (t *unixTail) setFile(f *os.File) { + if t.curFile != nil { + t.curFile.Close() + } + t.curFile = f + if f != nil { + err := unix.Fstat(int(f.Fd()), &t.curStat) + if err != nil { + f.Close() + fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) + return + } + t.reader = bufio.NewReaderSize(f, 1024*1024) + } else { + t.reader = nil + } +} + +// open log file and seek to the start of a line near the end, if possible. +func (t *unixTail) openTail() { + file, err := os.Open(t.fn) + if err != nil { + return + } + sz, err := file.Seek(0, os.SEEK_END) + if err != nil { + return + } + if sz < lastLinesSize { + sz, err = file.Seek(0, os.SEEK_SET) + if err == nil { + // just read from start of file, all OK + t.setFile(file) + } + return + } + sz, _ = file.Seek(-lastLinesSize, os.SEEK_END) + n, err := file.Read(t.lineBuf) + if err != nil { + return + } + for i := 0; i < n; i++ { + if t.lineBuf[i] == '\n' { + sz, err = file.Seek(sz+int64(i+1), os.SEEK_SET) + if err == nil { + t.setFile(file) + } + return + } + } +} + +func (t *unixTail) reopen(cur *unix.Stat_t) { + for cnt := 0; cnt < 100; cnt++ { + file, err := os.Open(t.fn) + if err != nil { + t.setFile(nil) + if cnt == 0 { + fmt.Fprintf(os.Stderr, "%v (waiting for log file to appear)\n", err) + } + time.Sleep(1000 * time.Millisecond) + continue + } + var stat unix.Stat_t + err = unix.Fstat(int(file.Fd()), &stat) + if err != nil { + file.Close() + fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) + time.Sleep(5000 * time.Millisecond) + continue + } + if cur != nil && cur.Dev == stat.Dev && cur.Ino == stat.Ino { + // same file, continue following it + file.Close() + return + } + // new file, start following it + t.setFile(file) + return + } +} + +// runs as a goroutine +func runTailWith(t *unixTail) { + defer t.setFile(nil) +loop: + for { + for t.curFile == nil { + t.reopen(nil) + } + bytes, err := t.reader.ReadSlice('\n') + t.lineBuf = append(t.lineBuf, bytes...) + if err == bufio.ErrBufferFull { + continue + } + if err == nil { + ll := len(t.lineBuf) - 1 + t.lines <- Line{Text: string(t.lineBuf[:ll])} + t.lineBuf = t.lineBuf[:0] + continue + } + if err == io.EOF { + pos, _ := t.curFile.Seek(0, os.SEEK_CUR) + for cnt := 0; cnt < 100; cnt++ { + time.Sleep(10 * time.Millisecond) + sz, _ := t.curFile.Seek(0, os.SEEK_END) + if sz != pos { + if sz < pos { + // truncation case + pos = 0 + } + t.curFile.Seek(pos, os.SEEK_SET) + continue loop + } + } + // no change in file size, try reopening + t.reopen(&t.curStat) + } else { + fmt.Fprintf(os.Stderr, "error tailing '%s': %v\n", t.fn, err) + close(t.lines) + return + } + } +} diff --git a/client/go/internal/admin/vespa-wrapper/main.go b/client/go/internal/admin/vespa-wrapper/main.go new file mode 100644 index 00000000000..eed8ac5b57b --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/main.go @@ -0,0 +1,107 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Entrypoint for internal Vespa commands: vespa-logfmt, vespa-deploy etc. +// Author: arnej + +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/clusterstate" + "github.com/vespa-engine/vespa/client/go/internal/admin/deploy" + "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" + "github.com/vespa-engine/vespa/client/go/internal/admin/vespa-wrapper/configserver" + "github.com/vespa-engine/vespa/client/go/internal/admin/vespa-wrapper/logfmt" + "github.com/vespa-engine/vespa/client/go/internal/admin/vespa-wrapper/services" + "github.com/vespa-engine/vespa/client/go/internal/admin/vespa-wrapper/standalone" + "github.com/vespa-engine/vespa/client/go/internal/admin/vespa-wrapper/startcbinary" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func basename(s string) string { + parts := strings.Split(s, "/") + return parts[len(parts)-1] +} + +func main() { + defer handleSimplePanic() + _ = vespa.FindAndVerifyVespaHome() + action := basename(os.Args[0]) + if action == "vespa-wrapper" && len(os.Args) > 1 { + action = os.Args[1] + os.Args = os.Args[1:] + } + switch action { + case "vespa-stop-services": + os.Exit(services.VespaStopServices()) + case "vespa-start-services": + os.Exit(services.VespaStartServices()) + case "start-services": + os.Exit(services.StartServices()) + case "just-run-configproxy": + os.Exit(services.JustRunConfigproxy()) + case "vespa-start-configserver": + os.Exit(configserver.StartConfigserverEtc()) + case "just-start-configserver": + os.Exit(configserver.JustStartConfigserver()) + case "vespa-start-container-daemon": + os.Exit(jvm.RunApplicationContainer(os.Args[1:])) + case "run-standalone-container": + os.Exit(standalone.StartStandaloneContainer(os.Args[1:])) + case "start-c-binary": + os.Exit(startcbinary.Run(os.Args[1:])) + case "export-env": + vespa.ExportDefaultEnvToSh() + case "security-env", "vespa-security-env": + vespa.ExportSecurityEnvToSh() + case "ipv6-only": + if vespa.HasOnlyIpV6() { + os.Exit(0) + } else { + os.Exit(1) + } + case "detect-hostname": + myName, err := vespa.FindOurHostname() + fmt.Println(myName) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + case "vespa-deploy": + cobra := deploy.NewDeployCmd() + cobra.Execute() + case "vespa-logfmt": + cobra := logfmt.NewLogfmtCmd() + cobra.Execute() + case "vespa-get-cluster-state": + cobra := clusterstate.NewGetClusterStateCmd() + cobra.Execute() + case "vespa-get-node-state": + cobra := clusterstate.NewGetNodeStateCmd() + cobra.Execute() + case "vespa-set-node-state": + cobra := clusterstate.NewSetNodeStateCmd() + cobra.Execute() + default: + if startcbinary.IsCandidate(os.Args[0]) { + os.Exit(startcbinary.Run(os.Args)) + } + fmt.Fprintf(os.Stderr, "unknown action '%s'\n", action) + fmt.Fprintln(os.Stderr, "actions: export-env, ipv6-only, security-env, detect-hostname") + fmt.Fprintln(os.Stderr, "(also: vespa-deploy, vespa-logfmt)") + } +} + +func handleSimplePanic() { + if r := recover(); r != nil { + if jee, ok := r.(*util.JustExitError); ok { + fmt.Fprintln(os.Stderr, jee) + os.Exit(1) + } else { + panic(r) + } + } +} diff --git a/client/go/internal/admin/vespa-wrapper/services/configproxy.go b/client/go/internal/admin/vespa-wrapper/services/configproxy.go new file mode 100644 index 00000000000..9807d8d4591 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/services/configproxy.go @@ -0,0 +1,143 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + CONFIGPROXY_PIDFILE = "var/run/configproxy.pid" + PROXY_SERVICE_NAME = "configproxy" +) + +func JustRunConfigproxy() int { + commonPreChecks() + vespa.CheckCorrectUser() + configsources := defaults.VespaConfigserverRpcAddrs() + if len(configsources) < 1 { + util.JustExitMsg("could not find any configservers") + } + util.TuneResourceLimits() + c := jvm.NewConfigProxyJvm(PROXY_SERVICE_NAME) + userargs := os.Getenv(envvars.VESPA_CONFIGPROXY_JVMARGS) + c.ConfigureOptions(configsources, userargs) + c.Exec() + // unreachable: + return 1 +} + +func startProxyWithRunserver() { + commonPreChecks() + vespa.CheckCorrectUser() + configsources := defaults.VespaConfigserverRpcAddrs() + fmt.Printf( + "Starting config proxy using %s as config source(s)\n", + strings.Join(configsources, " and ")) + args := []string{ + "-r", "10", + "-s", PROXY_SERVICE_NAME, + "-p", CONFIGPROXY_PIDFILE, + "--", + "libexec/vespa/vespa-wrapper", "just-run-configproxy", + } + cmd := exec.Command("vespa-runserver", args...) + cmd.Stdin = nil + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Start() + p := cmd.Process + if p != nil { + p.Release() + } +} + +func waitForProxyResponse() bool { + hname, _ := vespa.FindOurHostname() + backtick := util.BackTicksWithStderr + start := time.Now() + fmt.Printf("Waiting for config proxy to start\n") + for sleepcount := 0; sleepcount < 1800; sleepcount++ { + time.Sleep(100 * time.Millisecond) + got, err := os.ReadFile(CONFIGPROXY_PIDFILE) + if err == nil { + pid, err := strconv.Atoi(strings.TrimSpace(string(got))) + if err == nil && pid > 0 { + out, err := backtick.Run("vespa-ping-configproxy", "-s", hname) + if err == nil { + secs := time.Since(start).Seconds() + fmt.Printf("config proxy started after %ds (runserver pid %d)\n", int(secs), pid) + return true + } + if sleepcount%50 == 9 { + trace.Warning("Could not ping configproxy:", err) + if sleepcount%500 == 59 { + secs := time.Since(start).Seconds() + trace.Warning("ping output after", int(secs), "seconds:", strings.TrimSpace(out)) + logFile := defaults.VespaLogFile() + cmd := fmt.Sprintf("tail -n 15 %s | vespa-logfmt -l all -N", logFile) + out, err = backtick.Run("sh", "-c", cmd) + fmt.Fprintf(os.Stderr, "tail of logfile: >>>\n%s<<<\n", out) + } + } + } else { + trace.Debug("bad contents (", string(got), ") in pid file", CONFIGPROXY_PIDFILE) + } + } else { + trace.Debug("bad pid file", CONFIGPROXY_PIDFILE, err) + } + } + secs := time.Since(start).Seconds() + fmt.Fprintf(os.Stderr, "Config proxy still failed to start after %d seconds!\n", int(secs)) + got, err := os.ReadFile(CONFIGPROXY_PIDFILE) + if err != nil { + fmt.Fprintf(os.Stderr, "pid file %s was not created\n", CONFIGPROXY_PIDFILE) + return false + } + gotpid := strings.TrimSpace(string(got)) + pid, err := strconv.Atoi(gotpid) + if err != nil || pid < 1 { + fmt.Fprintf(os.Stderr, "invalid pid '%s' in file %s\n", gotpid, CONFIGPROXY_PIDFILE) + return false + } + out, err := backtick.Run("kill", "-0", gotpid) + if err != nil { + fmt.Fprintf(os.Stderr, "config proxy process `%s` has terminated: %s\n", gotpid, strings.TrimSpace(out)) + return false + } + out, err = backtick.Run("vespa-ping-configproxy", "-s", hname) + fmt.Fprintf(os.Stderr, "failed to ping configproxy: %s\n", out) + return false +} + +func StartConfigproxy() int { + commonPreChecks() + vespa.MaybeSwitchUser("start-configproxy") + startProxyWithRunserver() + if waitForProxyResponse() { + return 0 + } + return 1 +} + +func stopProxyWithRunserver() { + _, err := util.SystemCommand.Run("vespa-runserver", + "-s", PROXY_SERVICE_NAME, + "-p", CONFIGPROXY_PIDFILE, "-S") + if err != nil { + trace.Warning("Stopping sentinel:", err) + } +} diff --git a/client/go/internal/admin/vespa-wrapper/services/env.go b/client/go/internal/admin/vespa-wrapper/services/env.go new file mode 100644 index 00000000000..65dd583b766 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/services/env.go @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func exportSettings(vespaHome string) { + vlt := fmt.Sprintf("file:%s/logs/vespa/vespa.log", vespaHome) + lcd := fmt.Sprintf("%s/var/db/vespa/logcontrol", vespaHome) + dlp := fmt.Sprintf("%s/lib64:/opt/vespa-deps/lib64", vespaHome) + os.Setenv(envvars.VESPA_LOG_TARGET, vlt) + os.Setenv(envvars.VESPA_LOG_CONTROL_DIR, lcd) + os.Setenv(envvars.LD_LIBRARY_PATH, dlp) + os.Setenv(envvars.JAVAVM_LD_PRELOAD, "") + os.Setenv(envvars.LD_PRELOAD, "") + os.Setenv(envvars.MALLOC_ARENA_MAX, "1") + util.OptionallyReduceTimerFrequency() +} diff --git a/client/go/internal/admin/vespa-wrapper/services/prechecks.go b/client/go/internal/admin/vespa-wrapper/services/prechecks.go new file mode 100644 index 00000000000..690447a9fee --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/services/prechecks.go @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func commonPreChecks() (veHome, veHost string) { + if doTrace := os.Getenv(envvars.TRACE_STARTUP); doTrace != "" { + trace.AdjustVerbosity(1) + } + if doDebug := os.Getenv(envvars.DEBUG_STARTUP); doDebug != "" { + trace.AdjustVerbosity(2) + } + _ = vespa.FindAndVerifyVespaHome() + err := vespa.LoadDefaultEnv() + if err != nil { + panic(err) + } + veHome = vespa.FindAndVerifyVespaHome() + veHost, err = vespa.FindOurHostname() + if err != nil { + trace.Warning("could not detect hostname:", err, "; using fallback:", veHost) + } + err = os.Chdir(veHome) + if err != nil { + util.JustExitWith(err) + } + return +} diff --git a/client/go/internal/admin/vespa-wrapper/services/sentinel.go b/client/go/internal/admin/vespa-wrapper/services/sentinel.go new file mode 100644 index 00000000000..1017924263b --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/services/sentinel.go @@ -0,0 +1,93 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + SENTINEL_PIDFILE = "var/run/sentinel.pid" + SENTINEL_SERVICE_NAME = "config-sentinel" +) + +func startSentinelWithRunserver() { + _, veHost := commonPreChecks() + vespa.CheckCorrectUser() + os.Setenv(envvars.VESPA_SERVICE_NAME, SENTINEL_SERVICE_NAME) + configId := fmt.Sprintf("hosts/%s", veHost) + args := []string{ + "-r", "10", + "-s", SENTINEL_SERVICE_NAME, + "-p", SENTINEL_PIDFILE, + "--", + "sbin/vespa-config-sentinel", + "-c", configId, + } + cmd := exec.Command("vespa-runserver", args...) + cmd.Stdin = nil + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Start() + p := cmd.Process + if p != nil { + p.Release() + } + os.Unsetenv(envvars.VESPA_SERVICE_NAME) +} + +func waitForSentinelPid() bool { + backtick := util.BackTicksWithStderr + start := time.Now() + for sleepcount := 0; sleepcount < 1000; sleepcount++ { + time.Sleep(10 * time.Millisecond) + got, err := os.ReadFile(CONFIGPROXY_PIDFILE) + if err == nil { + pid, err := strconv.Atoi(strings.TrimSpace(string(got))) + if err == nil && pid > 0 { + _, err := backtick.Run("kill", "-0", strconv.Itoa(pid)) + if err == nil { + secs := int(time.Since(start).Seconds()) + if secs > 0 { + fmt.Printf("config sentinel started after %d seconds\n", secs) + } + return true + } + } + } + if sleepcount%500 == 90 { + trace.Warning("Waiting for sentinel to start") + } + } + return false +} + +func StartConfigSentinel() int { + commonPreChecks() + vespa.MaybeSwitchUser("start-config-sentinel") + startSentinelWithRunserver() + if waitForSentinelPid() { + return 0 + } + return 1 +} + +func stopSentinelWithRunserver() { + _, err := util.SystemCommand.Run("vespa-runserver", + "-s", SENTINEL_SERVICE_NAME, + "-p", SENTINEL_PIDFILE, "-S") + if err != nil { + trace.Warning("Stopping sentinel:", err) + } +} diff --git a/client/go/internal/admin/vespa-wrapper/services/start.go b/client/go/internal/admin/vespa-wrapper/services/start.go new file mode 100644 index 00000000000..f47d99714f1 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/services/start.go @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "fmt" + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func StartServices() int { + veHome, _ := commonPreChecks() + vespa.CheckCorrectUser() + trace.Debug("running as correct user") + exportSettings(veHome) + if vespa.HasOnlyIpV6() { + os.Setenv(envvars.VESPA_ONLY_IP_V6_NETWORKING, "true") + } + startProxyWithRunserver() + if waitForProxyResponse() { + startSentinelWithRunserver() + if waitForSentinelPid() { + return 0 + } + } + return 1 +} + +func checkjava() { + backticks := util.BackTicksWithStderr + out, err := backticks.Run("java", "-version") + if err != nil { + trace.Warning("cannot run 'java -version'") + util.JustExitWith(err) + } + if !strings.Contains(out, "64-Bit Server VM") { + util.JustExitWith(fmt.Errorf("java must invoke the 64-bit Java VM, but -version says:\n%s\n", out)) + } +} + +func runvalidation() { + // not implemented +} + +func VespaStartServices() int { + home, host := commonPreChecks() + trace.Debug("common prechecks ok, running in", home, "on", host) + vespa.RunPreStart() + trace.Debug("prestart ok") + util.TuneResourceLimits() + trace.Debug("resource limits ok") + checkjava() + trace.Debug("java ok") + runvalidation() + enable_transparent_hugepages_with_background_compaction() + disable_vm_zone_reclaim_mode() + drop_caches() + err := vespa.MaybeSwitchUser("start-services") + if err != nil { + util.JustExitWith(err) + } + return StartServices() +} diff --git a/client/go/internal/admin/vespa-wrapper/services/stop.go b/client/go/internal/admin/vespa-wrapper/services/stop.go new file mode 100644 index 00000000000..f5b764d122e --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/services/stop.go @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func VespaStopServices() int { + if doTrace := os.Getenv(envvars.TRACE_STARTUP); doTrace != "" { + trace.AdjustVerbosity(1) + } + if doDebug := os.Getenv(envvars.DEBUG_STARTUP); doDebug != "" { + trace.AdjustVerbosity(2) + } + err := vespa.LoadDefaultEnv() + if err != nil { + util.JustExitWith(err) + } + err = vespa.MaybeSwitchUser("vespa-stop-services") + if err != nil { + util.JustExitWith(err) + } + vespa.CheckCorrectUser() + trace.Debug("running as correct user") + stopSentinelWithRunserver() + stopProxyWithRunserver() + return 0 +} diff --git a/client/go/internal/admin/vespa-wrapper/services/tuning.go b/client/go/internal/admin/vespa-wrapper/services/tuning.go new file mode 100644 index 00000000000..f922495812f --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/services/tuning.go @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package services + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +const ( + DROP_CACHES_CTL = "/proc/sys/vm/drop_caches" + TRANSPARENT_HUGEPAGE_ENABLED = "/sys/kernel/mm/transparent_hugepage/enabled" + TRANSPARENT_HUGEPAGE_DEFRAG = "/sys/kernel/mm/transparent_hugepage/defrag" + TRANSPARENT_HUGEPAGE_KH_DEFRAG = "/sys/kernel/mm/transparent_hugepage/khugepaged/defrag" + ZONE_RECLAIM_CTL = "/proc/sys/vm/zone_reclaim_mode" +) + +func maybeEcho(fileName, toWrite string) bool { + f, err := os.OpenFile(fileName, os.O_WRONLY, 0) + if err == nil { + _, err = fmt.Fprintf(f, "%s\n", toWrite) + f.Close() + if err == nil { + trace.Debug("wrote", toWrite, "to", fileName) + return true + } else { + trace.Warning(err) + } + } + return false +} + +func enable_transparent_hugepages_with_background_compaction() { + // Should probably also be done on host. + maybeEcho(TRANSPARENT_HUGEPAGE_ENABLED, "always") + maybeEcho(TRANSPARENT_HUGEPAGE_DEFRAG, "never") + maybeEcho(TRANSPARENT_HUGEPAGE_KH_DEFRAG, "1") +} + +func disable_vm_zone_reclaim_mode() { + maybeEcho(ZONE_RECLAIM_CTL, "0") +} + +func drop_caches() { + if maybeEcho(DROP_CACHES_CTL, "3") { + trace.Debug("dropped caches") + } +} diff --git a/client/go/internal/admin/vespa-wrapper/standalone/start.go b/client/go/internal/admin/vespa-wrapper/standalone/start.go new file mode 100644 index 00000000000..a26ac8e9d8c --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/standalone/start.go @@ -0,0 +1,53 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +// for starting standalone jdisc containers +package standalone + +import ( + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func commonPreChecks() { + if doTrace := os.Getenv("TRACE_JVM_STARTUP"); doTrace != "" { + trace.AdjustVerbosity(1) + } + if doDebug := os.Getenv("DEBUG_JVM_STARTUP"); doDebug != "" { + trace.AdjustVerbosity(2) + } + veHome := vespa.FindAndVerifyVespaHome() + err := os.Chdir(veHome) + if err != nil { + util.JustExitWith(err) + } + err = vespa.LoadDefaultEnv() + if err != nil { + util.JustExitWith(err) + } +} + +func StartStandaloneContainer(extraArgs []string) int { + commonPreChecks() + util.TuneResourceLimits() + serviceName := os.Getenv("VESPA_SERVICE_NAME") + if serviceName == "" { + util.JustExitMsg("Missing service name, ensure VESPA_SERVICE_NAME is set in the environment") + } + c := jvm.NewStandaloneContainer(serviceName) + jvmOpts := c.JvmOptions() + jvmOpts.AddOption("-DOnnxBundleActivator.skip=true") + for _, extra := range extraArgs { + jvmOpts.AddOption(extra) + } + minFallback := jvm.MegaBytesOfMemory(128) + maxFallback := jvm.MegaBytesOfMemory(2048) + jvmOpts.AddDefaultHeapSizeArgs(minFallback, maxFallback) + c.Exec() + // unreachable: + return 1 +} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/cmd.go b/client/go/internal/admin/vespa-wrapper/startcbinary/cmd.go new file mode 100644 index 00000000000..9580a9240bb --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/cmd.go @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func Run(args []string) int { + trace.AdjustVerbosity(0) + if len(args) < 1 { + trace.Warning("missing program argument") + return 1 + } + spec := NewProgSpec(args) + err := vespa.LoadDefaultEnv() + if err != nil { + util.JustExitWith(err) + } + hostname, err := vespa.FindOurHostname() + if err != nil { + trace.Warning("could not detect hostname:", err, "; using fallback:", hostname) + } + return startCbinary(spec) +} + +func IsCandidate(program string) bool { + binary := program + "-bin" + if strings.Contains(binary, "/") { + return util.IsRegularFile(binary) + } else { + path := strings.Split(os.Getenv(envvars.PATH), ":") + for _, dir := range path { + fn := dir + "/" + binary + if util.IsRegularFile(fn) { + return true + } + } + } + return false +} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/common_env.go b/client/go/internal/admin/vespa-wrapper/startcbinary/common_env.go new file mode 100644 index 00000000000..6bc730b5119 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/common_env.go @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func (spec *ProgSpec) considerFallback(varName, varValue string) { + if spec.getenv(varName) == "" && varValue != "" { + spec.setenv(varName, varValue) + } +} + +func (spec *ProgSpec) considerEnvFallback(targetVar, fallbackVar string) { + spec.considerFallback(targetVar, spec.getenv(fallbackVar)) +} + +func (spec *ProgSpec) configureCommonEnv() { + os.Unsetenv(envvars.LD_PRELOAD) + spec.setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") + spec.setenv(envvars.GLIBCXX_FORCE_NEW, "1") + spec.setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") + spec.setenv(envvars.MALLOC_ARENA_MAX, "1") + + // fallback from old env.vars: + spec.considerEnvFallback(envvars.VESPA_USE_HUGEPAGES_LIST, envvars.HUGEPAGES_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_MADVISE_LIST, envvars.MADVISE_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC, envvars.VESPAMALLOC_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_D, envvars.VESPAMALLOCD_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_DST, envvars.VESPAMALLOCDST_LIST) + spec.considerEnvFallback(envvars.VESPA_USE_NO_VESPAMALLOC, envvars.NO_VESPAMALLOC_LIST) + // other fallbacks: + spec.considerFallback(envvars.ROOT, vespa.FindHome()) + spec.considerFallback(envvars.VESPA_USER, vespa.FindVespaUser()) + spec.considerFallback(envvars.VESPA_USE_HUGEPAGES_LIST, "all") + spec.considerFallback(envvars.VESPA_USE_VESPAMALLOC, "all") + spec.considerFallback(envvars.VESPA_USE_NO_VESPAMALLOC, strings.Join([]string{ + "vespa-rpc-invoke", + "vespa-get-config", + "vespa-sentinel-cmd", + "vespa-route", + "vespa-proton-cmd", + "vespa-configproxy-cmd", + "vespa-config-status", + }, " ")) + +} + +func (spec *ProgSpec) configureHugePages() { + if spec.matchesListEnv(envvars.VESPA_USE_HUGEPAGES_LIST) { + spec.setenv(envvars.VESPA_USE_HUGEPAGES, "yes") + } +} + +func (spec *ProgSpec) configureUseMadvise() { + limit := spec.valueFromListEnv(envvars.VESPA_USE_MADVISE_LIST) + if limit != "" { + trace.Trace("shall use madvise with limit", limit, "as set in", envvars.VESPA_USE_MADVISE_LIST) + spec.setenv(envvars.VESPA_MALLOC_MADVISE_LIMIT, limit) + return + } +} + +func (spec *ProgSpec) configurePath() { + // Prefer newer gdb and pstack: + spec.prependPath("/opt/rh/gcc-toolset-12/root/usr/bin") + // Maven is needed for tester applications: + spec.prependPath(vespa.FindHome() + "/local/maven/bin") + spec.prependPath(vespa.FindHome() + "/bin64") + spec.prependPath(vespa.FindHome() + "/bin") + // how to find the "java" program? + // should be available in $VESPA_HOME/bin or JAVA_HOME + if javaHome := spec.getenv(envvars.JAVA_HOME); javaHome != "" { + spec.prependPath(javaHome + "/bin") + } +} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/bad-numactl b/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/bad-numactl new file mode 100755 index 00000000000..1b08bb19b95 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/bad-numactl @@ -0,0 +1,25 @@ +#!/bin/sh + +# emulate numactl which says hardware is available but fails anyway + +case $#.$* in + 1.--hardware) + cat << 'EOF' +available: 2 nodes (0-1) +node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 +node 0 size: 128604 MB +node 0 free: 107167 MB +node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39 41 43 45 47 +node 1 size: 128976 MB +node 1 free: 110921 MB +node distances: +node 0 1 + 0: 10 21 + 1: 21 10 +EOF + ;; + *) + echo "numactl failure" + ;; +esac +exit 0 diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/good-numactl b/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/good-numactl new file mode 100755 index 00000000000..f861809b382 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/good-numactl @@ -0,0 +1,25 @@ +#!/bin/sh + +# emulate normal numactl operations + +case $#.$* in + 1.--hardware) + cat << 'EOF' +available: 2 nodes (0-1) +node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 +node 0 size: 128604 MB +node 0 free: 107167 MB +node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39 41 43 45 47 +node 1 size: 128976 MB +node 1 free: 110921 MB +node distances: +node 0 1 + 0: 10 21 + 1: 21 10 +EOF + ;; + *) + echo "foo" + ;; +esac +exit 0 diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/has-valgrind b/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/has-valgrind new file mode 100755 index 00000000000..8716209928e --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/has-valgrind @@ -0,0 +1,6 @@ +#!/bin/sh + +# emulate "which" that finds valgrind + +echo "/opt/vespa-deps/bin/valgrind" +exit 0 diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/no-numactl b/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/no-numactl new file mode 100755 index 00000000000..dfe583abaca --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/no-numactl @@ -0,0 +1,6 @@ +#!/bin/sh + +# emulate inoperational numactl + +echo "No NUMA available on this system" +exit 1 diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/no-valgrind b/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/no-valgrind new file mode 100755 index 00000000000..7a74e29b84b --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/mockbin/no-valgrind @@ -0,0 +1,6 @@ +#!/bin/sh + +# emulate "which" that does not find valgrind + +echo "which: no valgrind in ($PATH)" >&2 +exit 1 diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/numactl.go b/client/go/internal/admin/vespa-wrapper/startcbinary/numactl.go new file mode 100644 index 00000000000..fe091dedba9 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/numactl.go @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "fmt" + "strconv" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func (p *ProgSpec) configureNumaCtl() { + p.shouldUseNumaCtl = false + p.numaSocket = -1 + if p.getenv(envvars.VESPA_NO_NUMACTL) != "" { + return + } + backticks := util.BackTicksIgnoreStderr + out, err := backticks.Run("numactl", "--hardware") + trace.Debug("numactl --hardware says:", out) + if err != nil { + trace.Trace("numactl error:", err) + return + } + outfoo, errfoo := backticks.Run("numactl", "--interleave", "all", "echo", "foo") + if errfoo != nil { + trace.Trace("cannot run with numactl:", errfoo) + return + } + if outfoo != "foo\n" { + trace.Trace("bad numactl output:", outfoo) + return + } + p.shouldUseNumaCtl = true + if affinity := p.getenv(envvars.VESPA_AFFINITY_CPU_SOCKET); affinity != "" { + wantSocket, _ := strconv.Atoi(affinity) + trace.Debug("want socket:", wantSocket) + parts := strings.Fields(out) + for idx := 0; idx+2 < len(parts); idx++ { + if parts[idx] == "available:" && parts[idx+2] == "nodes" { + numSockets, _ := strconv.Atoi(parts[idx+1]) + trace.Debug("numSockets:", numSockets) + if numSockets > 1 { + p.numaSocket = (wantSocket % numSockets) + return + } + } + } + } +} + +func (p *ProgSpec) numaCtlBinary() string { + return "numactl" +} + +func (p *ProgSpec) prependNumaCtl(args []string) []string { + v := util.NewArrayList[string](5 + len(args)) + v.Append("numactl") + if p.numaSocket >= 0 { + v.Append(fmt.Sprintf("--cpunodebind=%d", p.numaSocket)) + v.Append(fmt.Sprintf("--membind=%d", p.numaSocket)) + } else { + v.Append("--interleave") + v.Append("all") + } + v.AppendAll(args...) + return v +} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/numactl_test.go b/client/go/internal/admin/vespa-wrapper/startcbinary/numactl_test.go new file mode 100644 index 00000000000..65f52be988e --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/numactl_test.go @@ -0,0 +1,89 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package startcbinary + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +func setup(t *testing.T, testFileName string) { + trace.AdjustVerbosity(1) + mockBinParent = strings.TrimSuffix(testFileName, "/numactl_test.go") + tmpBin = t.TempDir() + "/mock.bin.numactl_test" + err := os.MkdirAll(tmpBin, 0755) + assert.Nil(t, err) + t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH"))) +} + +func TestNumaCtlDetection(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + _, tfn, _, _ := runtime.Caller(0) + setup(t, tfn) + orig := []string{"/bin/myprog", "-c", "cfgid"} + spec := NewProgSpec(orig) + + useMock("no-numactl", "numactl") + spec.configureNumaCtl() + assert.Equal(t, false, spec.shouldUseNumaCtl) + + useMock("bad-numactl", "numactl") + spec.configureNumaCtl() + assert.Equal(t, false, spec.shouldUseNumaCtl) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "") + useMock("good-numactl", "numactl") + spec.configureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, -1, spec.numaSocket) + argv := spec.prependNumaCtl(orig) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "numactl", argv[0]) + assert.Equal(t, "--interleave", argv[1]) + assert.Equal(t, "all", argv[2]) + assert.Equal(t, "/bin/myprog-bin", argv[3]) + assert.Equal(t, "-c", argv[4]) + assert.Equal(t, "cfgid", argv[5]) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "0") + spec.configureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, 0, spec.numaSocket) + argv = spec.prependNumaCtl(orig) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "numactl", argv[0]) + assert.Equal(t, "--cpunodebind=0", argv[1]) + assert.Equal(t, "--membind=0", argv[2]) + assert.Equal(t, "/bin/myprog-bin", argv[3]) + assert.Equal(t, "-c", argv[4]) + assert.Equal(t, "cfgid", argv[5]) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "1") + spec.configureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, 1, spec.numaSocket) + argv = spec.prependNumaCtl(orig) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "numactl", argv[0]) + assert.Equal(t, "--cpunodebind=1", argv[1]) + assert.Equal(t, "--membind=1", argv[2]) + assert.Equal(t, "/bin/myprog-bin", argv[3]) + assert.Equal(t, "-c", argv[4]) + assert.Equal(t, "cfgid", argv[5]) + + t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "2") + spec.configureNumaCtl() + assert.Equal(t, true, spec.shouldUseNumaCtl) + assert.Equal(t, 0, spec.numaSocket) + +} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/progspec.go b/client/go/internal/admin/vespa-wrapper/startcbinary/progspec.go new file mode 100644 index 00000000000..9975f6c3c90 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/progspec.go @@ -0,0 +1,144 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" +) + +type ProgSpec struct { + Program string + Args []string + BaseName string + Env map[string]string + numaSocket int + shouldUseCallgrind bool + shouldUseValgrind bool + shouldUseNumaCtl bool + shouldUseVespaMalloc bool + vespaMallocPreload string +} + +func NewProgSpec(argv []string) *ProgSpec { + progName := argv[0] + binProg := progName + "-bin" + p := ProgSpec{ + Program: binProg, + Args: argv, + BaseName: baseNameOf(progName), + Env: make(map[string]string), + numaSocket: -1, + } + p.Args[0] = binProg + return &p +} + +func baseNameOf(s string) string { + idx := strings.LastIndex(s, "/") + idx++ + return s[idx:] +} + +func (p *ProgSpec) setenv(k, v string) { + p.Env[k] = v +} + +func (p *ProgSpec) getenv(k string) string { + if v, ok := p.Env[k]; ok { + return v + } + return os.Getenv(k) +} + +func (p *ProgSpec) prependPath(dirName string) { + pathList := []string{dirName} + oldPath := p.getenv(envvars.PATH) + if oldPath == "" { + oldPath = "/usr/bin" + } + for _, part := range strings.Split(oldPath, ":") { + if part != dirName { + pathList = append(pathList, part) + } + } + newPath := strings.Join(pathList, ":") + p.setenv(envvars.PATH, newPath) + os.Setenv(envvars.PATH, newPath) +} + +func (p *ProgSpec) matchesListEnv(envVarName string) bool { + return p.matchesListString(p.getenv(envVarName)) +} + +func (p *ProgSpec) matchesListString(env string) bool { + if env == "all" { + trace.Debug(p.Program, "always matching in:", env) + return true + } + parts := strings.Fields(env) + for _, part := range parts { + if p.BaseName == part { + trace.Debug(p.Program, "has basename matching in:", env) + return true + } + trace.Debug("checking matching:", p.BaseName, "!=", part) + } + return false +} + +func (p *ProgSpec) valueFromListEnv(envVarName string) string { + return p.valueFromListString(p.getenv(envVarName)) +} + +func (p *ProgSpec) valueFromListString(env string) string { + parts := strings.Fields(env) + for _, part := range parts { + idx := strings.Index(part, "=") + if idx <= 0 { + trace.Trace("expected key=value, but got:", part) + continue + } + partName := part[:idx] + idx++ + partValue := part[idx:] + if p.BaseName == partName || partName == "all" { + trace.Debug(p.Program, "has basename matching in:", env) + return partValue + } + trace.Debug("checking matching:", p.BaseName, "!=", part) + } + return "" +} + +func (spec *ProgSpec) effectiveEnv() []string { + env := make(map[string]string) + for _, entry := range os.Environ() { + addInMap := func(kv string) bool { + for idx, elem := range kv { + if elem == '=' { + k := kv[:idx] + env[k] = kv + return true + } + } + return false + } + if !addInMap(entry) { + env[entry] = "" + } + } + for k, v := range spec.Env { + trace.Trace("add to environment:", k, "=", v) + env[k] = k + "=" + v + } + envv := make([]string, 0, len(env)) + for _, v := range env { + envv = append(envv, v) + } + return envv +} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/progspec_test.go b/client/go/internal/admin/vespa-wrapper/startcbinary/progspec_test.go new file mode 100644 index 00000000000..be113e4e350 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/progspec_test.go @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package startcbinary + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProgSpec(t *testing.T) { + spec := NewProgSpec([]string{"/opt/vespa/bin/foobar"}) + var b bool + + b = spec.matchesListString("") + assert.Equal(t, false, b) + b = spec.matchesListString("foobar") + assert.Equal(t, true, b) + b = spec.matchesListString("foo bar") + assert.Equal(t, false, b) + b = spec.matchesListString("one foobar") + assert.Equal(t, true, b) + b = spec.matchesListString("foobar two") + assert.Equal(t, true, b) + b = spec.matchesListString("one foobar two") + assert.Equal(t, true, b) + b = spec.matchesListString("all") + assert.Equal(t, true, b) + + var s string + s = spec.valueFromListString("") + assert.Equal(t, "", s) + s = spec.valueFromListString("foobar=123") + assert.Equal(t, "123", s) + s = spec.valueFromListString("one=1 foobar=123 two=2") + assert.Equal(t, "123", s) + s = spec.valueFromListString("one=1 all=123") + assert.Equal(t, "123", s) +} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/startcbinary.go b/client/go/internal/admin/vespa-wrapper/startcbinary/startcbinary.go new file mode 100644 index 00000000000..f5e58e59808 --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/startcbinary.go @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func startCbinary(spec *ProgSpec) int { + spec.configureCommonEnv() + spec.configurePath() + spec.configureTuning() + spec.configureValgrind() + spec.configureNumaCtl() + spec.configureHugePages() + spec.configureUseMadvise() + spec.configureVespaMalloc() + err := spec.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } else { + return 0 + } +} + +func (spec *ProgSpec) run() error { + prog := spec.Program + args := spec.Args + if spec.shouldUseValgrind { + args = spec.prependValgrind(args) + prog = spec.valgrindBinary() + } else if spec.shouldUseNumaCtl { + args = spec.prependNumaCtl(args) + prog = spec.numaCtlBinary() + } + if spec.shouldUseVespaMalloc { + spec.setenv(envvars.LD_PRELOAD, spec.vespaMallocPreload) + } + envv := spec.effectiveEnv() + return util.Execvpe(prog, args, envv) +} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go b/client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go new file mode 100644 index 00000000000..57230629d7a --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +func (spec *ProgSpec) configureTuning() { + util.OptionallyReduceTimerFrequency() + util.TuneResourceLimits() +} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go new file mode 100644 index 00000000000..43a1ed602bd --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go @@ -0,0 +1,82 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "fmt" + "os" + "strings" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func (p *ProgSpec) configureValgrind() { + p.shouldUseValgrind = false + p.shouldUseCallgrind = false + env := p.getenv(envvars.VESPA_USE_VALGRIND) + parts := strings.Split(env, " ") + for _, part := range parts { + if p.BaseName == part { + trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env) + backticks := util.BackTicksWithStderr + out, err := backticks.Run("which", "valgrind") + if err != nil { + trace.Trace("no valgrind, 'which' fails:", err, "=>", out) + return + } + if opts := p.getenv(envvars.VESPA_VALGRIND_OPT); strings.Contains(opts, "callgrind") { + p.shouldUseCallgrind = true + } + p.shouldUseValgrind = true + return + } + trace.Debug("checking", envvars.VESPA_USE_VALGRIND, ":", p.BaseName, "!=", part) + } +} + +func (p *ProgSpec) valgrindBinary() string { + return "valgrind" +} + +func (p *ProgSpec) valgrindOptions() []string { + env := p.getenv(envvars.VESPA_VALGRIND_OPT) + if env != "" { + return strings.Fields(env) + } + result := []string{ + "--num-callers=32", + "--run-libc-freeres=yes", + "--track-origins=yes", + "--freelist-vol=1000000000", + "--leak-check=full", + "--show-reachable=yes", + } + result = addValgrindSuppression(result, "etc/vespa/valgrind-suppressions.txt") + result = addValgrindSuppression(result, "etc/vespa/suppressions.txt") + return result +} + +func addValgrindSuppression(r []string, fn string) []string { + existsOk, fileName := vespa.HasFileUnderVespaHome(fn) + if existsOk { + r = append(r, fmt.Sprintf("--suppressions=%s", fileName)) + } + return r +} + +func (p *ProgSpec) valgrindLogOption() string { + return fmt.Sprintf("--log-file=%s/tmp/valgrind.%s.log.%d", vespa.FindHome(), p.BaseName, os.Getpid()) +} + +func (p *ProgSpec) prependValgrind(args []string) []string { + v := util.NewArrayList[string](15 + len(args)) + v.Append(p.valgrindBinary()) + v.AppendAll(p.valgrindOptions()...) + v.Append(p.valgrindLogOption()) + v.AppendAll(args...) + return v +} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go new file mode 100644 index 00000000000..48cc78474ed --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package startcbinary + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" +) + +var tmpBin string +var mockBinParent string + +func useMock(prog, target string) { + mock := fmt.Sprintf("%s/mockbin/%s", mockBinParent, prog) + symlink := fmt.Sprintf("%s/%s", tmpBin, target) + os.Remove(symlink) + err := os.Symlink(mock, symlink) + if err != nil { + util.JustExitWith(err) + } +} + +func setupValgrind(t *testing.T, testFileName string) { + trace.AdjustVerbosity(1) + t.Setenv("VESPA_HOME", mockBinParent+"/mock_vespahome") + mockBinParent = strings.TrimSuffix(testFileName, "/valgrind_test.go") + tmpBin = t.TempDir() + "/mock.bin.valgrind_test" + err := os.MkdirAll(tmpBin, 0755) + assert.Nil(t, err) + t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH"))) +} + +func TestValgrindDetection(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + _, tfn, _, _ := runtime.Caller(0) + setupValgrind(t, tfn) + spec := NewProgSpec([]string{"/opt/vespa/bin/foobar"}) + var argv []string + + useMock("has-valgrind", "which") + + t.Setenv("VESPA_USE_VALGRIND", "") + spec.configureValgrind() + assert.Equal(t, false, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + t.Setenv("VESPA_USE_VALGRIND", "foo bar") + spec.configureValgrind() + assert.Equal(t, false, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + t.Setenv("VESPA_USE_VALGRIND", "foobar") + spec.configureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"}) + trace.Trace("argv:", argv) + assert.Equal(t, 11, len(argv)) + assert.Equal(t, "valgrind", argv[0]) + assert.Equal(t, "/bin/myprog", argv[8]) + + t.Setenv("VESPA_USE_VALGRIND", "another foobar yetmore") + spec.configureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + + t.Setenv("VESPA_VALGRIND_OPT", "--tool=callgrind") + spec.configureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, true, spec.shouldUseCallgrind) + + argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"}) + trace.Trace("argv:", argv) + assert.Equal(t, 6, len(argv)) + assert.Equal(t, "valgrind", argv[0]) + assert.Equal(t, "--tool=callgrind", argv[1]) + assert.Equal(t, "/bin/myprog", argv[3]) + + useMock("no-valgrind", "which") + spec.configureValgrind() + assert.Equal(t, false, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) +} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/vespamalloc.go b/client/go/internal/admin/vespa-wrapper/startcbinary/vespamalloc.go new file mode 100644 index 00000000000..c6d53e1d03c --- /dev/null +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/vespamalloc.go @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package startcbinary + +import ( + "fmt" + + "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +func vespaMallocLib(suf string) string { + prefixes := []string{"lib64", "lib"} + for _, pre := range prefixes { + fn := fmt.Sprintf("%s/vespa/malloc/%s", pre, suf) + existsOk, fileName := vespa.HasFileUnderVespaHome(fn) + if existsOk { + trace.Debug("found library:", fileName) + return fileName + } + trace.Debug("bad or missing library:", fn) + } + return "" +} + +func (p *ProgSpec) configureVespaMalloc() { + p.shouldUseVespaMalloc = false + if p.matchesListEnv(envvars.VESPA_USE_NO_VESPAMALLOC) { + trace.Trace("use no vespamalloc:", p.BaseName) + return + } + if p.shouldUseValgrind && !p.shouldUseCallgrind { + trace.Trace("use valgrind, so no vespamalloc:", p.BaseName) + return + } + var useFile string + if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_DST) { + useFile = vespaMallocLib("libvespamallocdst16.so") + } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_D) { + useFile = vespaMallocLib("libvespamallocd.so") + } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC) { + useFile = vespaMallocLib("libvespamalloc.so") + } + trace.Trace("use file:", useFile) + if useFile == "" { + return + } + if loadAsHuge := p.getenv(envvars.VESPA_LOAD_CODE_AS_HUGEPAGES); loadAsHuge != "" { + otherFile := vespaMallocLib("libvespa_load_as_huge.so") + useFile = fmt.Sprintf("%s:%s", useFile, otherFile) + } + p.considerEnvFallback(envvars.VESPA_MALLOC_HUGEPAGES, envvars.VESPA_USE_HUGEPAGES) + p.vespaMallocPreload = useFile + p.shouldUseVespaMalloc = true +} diff --git a/client/go/internal/cli/cmd/vespa/main.go b/client/go/internal/cli/cmd/vespa/main.go index cca35ba8368..ade27680230 100644 --- a/client/go/internal/cli/cmd/vespa/main.go +++ b/client/go/internal/cli/cmd/vespa/main.go @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Cobra commands main file +// Entrypoint for Vespa CLI // Author: bratseth package main diff --git a/client/go/internal/vespa/find_home.go b/client/go/internal/vespa/find_home.go index 672d3d18724..89b5f5a7e21 100644 --- a/client/go/internal/vespa/find_home.go +++ b/client/go/internal/vespa/find_home.go @@ -16,7 +16,7 @@ import ( const ( defaultVespaInstallDir = "/opt/vespa" - scriptUtilsFilename = "libexec/vespa/script-utils" + scriptUtilsFilename = "libexec/vespa/vespa-wrapper" ) func FindHome() string { diff --git a/client/go/internal/vespa/switch_user.go b/client/go/internal/vespa/switch_user.go index 01e8794953d..ccfe7c4e7c3 100644 --- a/client/go/internal/vespa/switch_user.go +++ b/client/go/internal/vespa/switch_user.go @@ -58,7 +58,7 @@ func CheckCorrectUser() { util.JustExitMsg("running as wrong user. Check your VESPA_USER setting") } -// re-execute a script-utils action after switching to the vespa user +// re-execute a vespa-wrapper action after switching to the vespa user // (used by vespa-start-configserver and vespa-start-services) func MaybeSwitchUser(action string) error { const SU_PROG = "vespa-run-as-vespa-user" -- cgit v1.2.3