diff options
161 files changed, 909 insertions, 877 deletions
diff --git a/client/go/internal/admin/clusterstate/cluster_state.go b/client/go/internal/admin/clusterstate/cluster_state.go index 52f7c2181b4..fe6c6c1a4d2 100644 --- a/client/go/internal/admin/clusterstate/cluster_state.go +++ b/client/go/internal/admin/clusterstate/cluster_state.go @@ -11,7 +11,7 @@ import ( "strings" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) // common struct used various places in the clustercontroller REST api: @@ -117,6 +117,6 @@ func (model *VespaModelConfig) getClusterState(cluster string) (*ClusterState, * return &parsedJson, &cc } // no success: - util.JustExitMsg(fmt.Sprint(errs)) + osutil.ExitMsg(fmt.Sprint(errs)) panic("unreachable") } diff --git a/client/go/internal/admin/clusterstate/detect_model.go b/client/go/internal/admin/clusterstate/detect_model.go index 0f261526145..db2e167ea6a 100644 --- a/client/go/internal/admin/clusterstate/detect_model.go +++ b/client/go/internal/admin/clusterstate/detect_model.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -17,11 +17,11 @@ func getConfigServerHosts(s string) []string { if s != "" { return []string{s} } - backticks := util.BackTicksForwardStderr + backticks := osutil.BackTicksForwardStderr got, err := backticks.Run(vespa.FindHome()+"/bin/vespa-print-default", "configservers") res := strings.Fields(got) if err != nil || len(res) < 1 { - util.JustExitMsg("bad configservers: " + got) + osutil.ExitMsg("bad configservers: " + got) } trace.Debug("found", len(res), "configservers:", res) return res @@ -31,13 +31,13 @@ func getConfigServerPort(i int) int { if i > 0 { return i } - backticks := util.BackTicksForwardStderr + backticks := osutil.BackTicksForwardStderr got, err := backticks.Run(vespa.FindHome()+"/bin/vespa-print-default", "configserver_rpc_port") if err == nil { i, err = strconv.Atoi(strings.TrimSpace(got)) } if err != nil || i < 1 { - util.JustExitMsg("bad configserver_rpc_port: " + got) + osutil.ExitMsg("bad configserver_rpc_port: " + got) } trace.Debug("found configservers rpc port:", i) return i @@ -55,13 +55,13 @@ func detectModel(opts *Options) *VespaModelConfig { "-p", strconv.Itoa(cfgPort), "-s", cfgHost, } - backticks := util.BackTicksForwardStderr + backticks := osutil.BackTicksForwardStderr data, err := backticks.Run(vespa.FindHome()+"/bin/vespa-get-config", args...) parsed := parseModelConfig(data) if err == nil && parsed != nil { return parsed } } - util.JustExitMsg("could not get model config") + osutil.ExitMsg("could not get model config") panic("unreachable") } diff --git a/client/go/internal/admin/clusterstate/set_node_state.go b/client/go/internal/admin/clusterstate/set_node_state.go index a838b43503f..023d3ed71c8 100644 --- a/client/go/internal/admin/clusterstate/set_node_state.go +++ b/client/go/internal/admin/clusterstate/set_node_state.go @@ -16,7 +16,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" "github.com/vespa-engine/vespa/client/go/internal/build" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) const ( @@ -80,7 +80,7 @@ func runSetNodeState(opts *Options, args []string) { } wanted, err := knownState(args[0]) if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } reason := "" if len(args) > 1 { @@ -138,7 +138,7 @@ func (cc *ClusterControllerSpec) setNodeUserState(s serviceSpec, wanted KnownSta } jsonBytes, err := json.Marshal(request) if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } url := fmt.Sprintf("http://%s:%d/cluster/v2/%s/%s/%d", cc.host, cc.port, diff --git a/client/go/internal/admin/deploy/cmd.go b/client/go/internal/admin/deploy/cmd.go index 955ae93f859..f2edf3c7450 100644 --- a/client/go/internal/admin/deploy/cmd.go +++ b/client/go/internal/admin/deploy/cmd.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" "github.com/vespa-engine/vespa/client/go/internal/build" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -24,7 +24,7 @@ func NewDeployCmd() *cobra.Command { curOptions Options ) if err := vespa.LoadDefaultEnv(); err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } cobra.EnableCommandSorting = false cmd := &cobra.Command{ diff --git a/client/go/internal/admin/deploy/curl.go b/client/go/internal/admin/deploy/curl.go index 0ce1305226f..accd16b06f5 100644 --- a/client/go/internal/admin/deploy/curl.go +++ b/client/go/internal/admin/deploy/curl.go @@ -14,7 +14,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/trace" "github.com/vespa-engine/vespa/client/go/internal/curl" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -60,20 +60,20 @@ func urlWithoutQuery(url string) string { func newCurlCommand(url string, args []string) *curl.Command { tls, err := vespa.LoadTlsConfig() if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } if tls != nil && strings.HasPrefix(url, "http:") { url = "https:" + url[5:] } cmd, err := curl.RawArgs(url, args...) if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } if tls != nil { if tls.DisableHostnameValidation { cmd, err = curl.RawArgs(url, append(args, "--insecure")...) if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } } cmd.PrivateKey = tls.Files.PrivateKey diff --git a/client/go/internal/admin/deploy/fetch.go b/client/go/internal/admin/deploy/fetch.go index 9b4a927882d..010c0b1f324 100644 --- a/client/go/internal/admin/deploy/fetch.go +++ b/client/go/internal/admin/deploy/fetch.go @@ -12,7 +12,7 @@ import ( "strconv" "strings" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) // main entry point for vespa-deploy fetch @@ -90,7 +90,7 @@ func getPartAfterSlash(path string) string { return parts[idx-1] } if idx == 0 { - util.JustExitMsg("cannot find part after slash: " + path) + osutil.ExitMsg("cannot find part after slash: " + path) } return parts[idx] } diff --git a/client/go/internal/admin/deploy/persist.go b/client/go/internal/admin/deploy/persist.go index 2c40606e1ab..781fc64106a 100644 --- a/client/go/internal/admin/deploy/persist.go +++ b/client/go/internal/admin/deploy/persist.go @@ -10,7 +10,7 @@ import ( "path/filepath" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) const ( @@ -43,11 +43,11 @@ func configsourceUrlUsedFile() string { func createTenantDir(tenant string) string { vespaDeployTempDir, err := createVespaDeployDir() if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } tdir := filepath.Join(vespaDeployTempDir, tenant) if err := os.MkdirAll(tdir, 0700); err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } return tdir } @@ -80,7 +80,7 @@ func getSessionIdFromFile(tenant string) string { fn := filepath.Join(dir, sessionIdFileName) bytes, err := os.ReadFile(fn) if err != nil { - util.JustExitMsg("Could not read session id from file, and no session id supplied as argument. Exiting.") + osutil.ExitMsg("Could not read session id from file, and no session id supplied as argument. Exiting.") } trace.Trace("Session-id", string(bytes), "found from file", fn) return string(bytes) diff --git a/client/go/internal/admin/deploy/urls.go b/client/go/internal/admin/deploy/urls.go index e9110bd9463..e840f4de29c 100644 --- a/client/go/internal/admin/deploy/urls.go +++ b/client/go/internal/admin/deploy/urls.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -28,7 +28,7 @@ func makeConfigsourceUrls(opts *Options) []string { var results = make([]string, 0, 3) if opts.ServerHost == "" { home := vespa.FindHome() - backticks := util.BackTicksForwardStderr + backticks := osutil.BackTicksForwardStderr configsources, _ := backticks.Run(home+"/bin/vespa-print-default", "configservers_http") for _, src := range strings.Split(configsources, "\n") { colonParts := strings.Split(src, ":") diff --git a/client/go/internal/admin/jvm/application_container.go b/client/go/internal/admin/jvm/application_container.go index c525ea77949..3806dfea302 100644 --- a/client/go/internal/admin/jvm/application_container.go +++ b/client/go/internal/admin/jvm/application_container.go @@ -4,14 +4,16 @@ package jvm import ( + "crypto/md5" "fmt" + "io" "os" "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/prog" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) const ( @@ -22,6 +24,13 @@ type ApplicationContainer struct { containerBase } +func md5Hex(text string) string { + hasher := md5.New() + io.WriteString(hasher, text) + hash := hasher.Sum(nil) + return fmt.Sprintf("%x", hash) +} + func (a *ApplicationContainer) ArgForMain() string { dir := defaults.UnderVespaHome("lib/jars") return fmt.Sprintf("file:%s/%s", dir, JAR_FOR_APPLICATION_CONTAINER) @@ -31,7 +40,7 @@ func (a *ApplicationContainer) Discriminator() string { cfgId := a.ConfigId() if cfgId != "" { trace.Trace("Discriminator: using md5 of", cfgId) - return util.Md5Hex(cfgId + "\n") + return md5Hex(cfgId + "\n") } svcName := a.ServiceName() if svcName != "" { @@ -40,7 +49,7 @@ func (a *ApplicationContainer) Discriminator() string { } pid := os.Getpid() trace.Trace("Discriminator: using md5 of", pid) - return util.Md5Hex(fmt.Sprintf("%d", pid)) + return md5Hex(fmt.Sprintf("%d", pid)) } func (a *ApplicationContainer) addJdiscProperties() { @@ -168,6 +177,6 @@ func (c *ApplicationContainer) exportExtraEnv(ps *prog.Spec) { if c.ConfigId() != "" { ps.Setenv(envvars.VESPA_CONFIG_ID, c.ConfigId()) } else { - util.JustExitMsg("application container requires a config id") + osutil.ExitMsg("application container requires a config id") } } diff --git a/client/go/internal/util/md5_test.go b/client/go/internal/admin/jvm/application_container_test.go index ac3bc6a9546..3a062edee16 100644 --- a/client/go/internal/util/md5_test.go +++ b/client/go/internal/admin/jvm/application_container_test.go @@ -1,5 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package util +// Author: arnej + +package jvm import ( "testing" @@ -8,6 +10,6 @@ import ( ) func TestMD5SimpleInputs(t *testing.T) { - assert.Equal(t, "d41d8cd98f00b204e9800998ecf8427e", Md5Hex("")) - assert.Equal(t, "4044e8209f286312a68bbb54f8714922", Md5Hex("admin/cluster-controllers/0\n")) + assert.Equal(t, "d41d8cd98f00b204e9800998ecf8427e", md5Hex("")) + assert.Equal(t, "4044e8209f286312a68bbb54f8714922", md5Hex("admin/cluster-controllers/0\n")) } diff --git a/client/go/internal/admin/jvm/container.go b/client/go/internal/admin/jvm/container.go index 086ca8dde25..852bdb2464b 100644 --- a/client/go/internal/admin/jvm/container.go +++ b/client/go/internal/admin/jvm/container.go @@ -10,7 +10,8 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/prog" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/list" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) type Container interface { @@ -60,7 +61,7 @@ func readableEnv(env map[string]string) string { } func (cb *containerBase) Exec() { - argv := util.ArrayListOf(cb.JvmOptions().Args()) + argv := list.ArrayListOf(cb.JvmOptions().Args()) argv.Insert(0, "java") p := prog.NewSpec(argv) p.ConfigureNumaCtl() @@ -71,5 +72,5 @@ func (cb *containerBase) Exec() { trace.Info("JVM env:", readableEnv(p.Env)) trace.Info("JVM exec:", argv) err := p.Run() - util.JustExitWith(err) + osutil.ExitErr(err) } diff --git a/client/go/internal/admin/jvm/env.go b/client/go/internal/admin/jvm/env.go index 5c1e938d46b..ab7b005668b 100644 --- a/client/go/internal/admin/jvm/env.go +++ b/client/go/internal/admin/jvm/env.go @@ -11,7 +11,8 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/prog" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) func (opts *Options) exportEnvSettings(ps *prog.Spec) { @@ -33,7 +34,7 @@ func (opts *Options) exportEnvSettings(ps *prog.Spec) { if preload := ps.Getenv(envvars.PRELOAD); preload != "" { checked := []string{} for _, fileName := range strings.Split(preload, ":") { - if util.PathExists(fileName) { + if ioutil.Exists(fileName) { checked = append(checked, fileName) } else { trace.Info("File in PRELOAD missing, skipped:", fileName) @@ -45,6 +46,6 @@ func (opts *Options) exportEnvSettings(ps *prog.Spec) { ps.Setenv(envvars.LD_PRELOAD, preload) } } - util.OptionallyReduceTimerFrequency() + osutil.OptionallyReduceTimerFrequency() c.exportExtraEnv(ps) } diff --git a/client/go/internal/admin/jvm/mem_avail.go b/client/go/internal/admin/jvm/mem_avail.go index bcc475e8ba8..df5acf79043 100644 --- a/client/go/internal/admin/jvm/mem_avail.go +++ b/client/go/internal/admin/jvm/mem_avail.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) func parseFree(txt string) AmountOfMemory { @@ -91,7 +91,7 @@ func vespa_cg2get_impl(rootdir, limitname string) (output string, err error) { func getAvailableMemory() AmountOfMemory { result := BytesOfMemory(0) - backticks := util.BackTicksWithStderr + backticks := osutil.BackTicksWithStderr freeOutput, err := backticks.Run("free", "-m") if err == nil { result = parseFree(freeOutput) diff --git a/client/go/internal/admin/jvm/options.go b/client/go/internal/admin/jvm/options.go index a788fb0cca9..30c07526a40 100644 --- a/client/go/internal/admin/jvm/options.go +++ b/client/go/internal/admin/jvm/options.go @@ -10,22 +10,23 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/list" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) type Options struct { container Container classPath []string - jvmArgs util.ArrayList[string] + jvmArgs list.ArrayList[string] mainClass string jarWithDeps string - fixSpec util.FixSpec + fixSpec osutil.FixSpec } func NewOptions(c Container) *Options { vespaUid, vespaGid := vespa.FindVespaUidAndGid() - fixSpec := util.FixSpec{ + fixSpec := osutil.FixSpec{ UserId: vespaUid, GroupId: vespaGid, DirMode: 0755, @@ -79,7 +80,7 @@ func (opts *Options) AddJvmArgsFromString(args string) { func (opts *Options) ConfigureCpuCount(cnt int) { if cnt <= 0 { - out, err := util.BackTicksForwardStderr.Run("nproc", "--all") + out, err := osutil.BackTicksForwardStderr.Run("nproc", "--all") if err != nil { trace.Trace("failed nproc:", err) } else { diff --git a/client/go/internal/admin/jvm/properties.go b/client/go/internal/admin/jvm/properties.go index 446f218a2ad..d0016e03b51 100644 --- a/client/go/internal/admin/jvm/properties.go +++ b/client/go/internal/admin/jvm/properties.go @@ -11,7 +11,7 @@ import ( "strings" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) // quote as specified in JDK source file java.base/share/classes/java/util/Properties.java @@ -101,6 +101,6 @@ func writeEnvAsProperties(envv []string, propsFile string) { trace.Trace("write props file:", propsFile) err := os.WriteFile(propsFile, envAsProperties(envv), 0600) if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } } diff --git a/client/go/internal/admin/jvm/qr_start_cfg.go b/client/go/internal/admin/jvm/qr_start_cfg.go index 4edb02b2a84..a14ab0d1946 100644 --- a/client/go/internal/admin/jvm/qr_start_cfg.go +++ b/client/go/internal/admin/jvm/qr_start_cfg.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) type QrStartConfig struct { @@ -42,10 +42,10 @@ func (a *ApplicationContainer) getQrStartCfg() *QrStartConfig { "-n", "search.config.qr-start", "-i", a.ConfigId(), } - backticks := util.BackTicksForwardStderr + backticks := osutil.BackTicksForwardStderr data, err := backticks.Run("vespa-get-config", args...) if err != nil { - util.JustExitMsg("could not get qr-start config: " + err.Error()) + osutil.ExitMsg("could not get qr-start config: " + err.Error()) } else { codec := json.NewDecoder(strings.NewReader(data)) err = codec.Decode(&parsedJson) diff --git a/client/go/internal/admin/jvm/standalone_container.go b/client/go/internal/admin/jvm/standalone_container.go index 859aea9157d..20031bc7725 100644 --- a/client/go/internal/admin/jvm/standalone_container.go +++ b/client/go/internal/admin/jvm/standalone_container.go @@ -9,7 +9,8 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/prog" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) const ( @@ -78,9 +79,9 @@ func (a *StandaloneContainer) addJdiscProperties() { func (c *StandaloneContainer) exportExtraEnv(ps *prog.Spec) { vespaHome := defaults.VespaHome() app := fmt.Sprintf("%s/conf/%s-app", vespaHome, c.ServiceName()) - if util.IsDirectory(app) { + if ioutil.IsDir(app) { ps.Setenv(envvars.STANDALONE_JDISC_APP_LOCATION, app) } else { - util.JustExitMsg("standalone container requires an application directory, missing: " + app) + osutil.ExitMsg("standalone container requires an application directory, missing: " + app) } } diff --git a/client/go/internal/admin/jvm/zk_locks.go b/client/go/internal/admin/jvm/zk_locks.go index 90a05f94905..3f0c8ea3301 100644 --- a/client/go/internal/admin/jvm/zk_locks.go +++ b/client/go/internal/admin/jvm/zk_locks.go @@ -8,7 +8,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) const ( @@ -16,12 +16,12 @@ const ( ) func RemoveStaleZkLocks(c Container) { - backticks := util.BackTicksWithStderr + backticks := osutil.BackTicksWithStderr cmd := fmt.Sprintf("rm -f '%s/%s.%s'*lck", defaults.VespaHome(), ZOOKEEPER_LOG_FILE_PREFIX, c.ServiceName()) trace.Trace("cleaning locks:", cmd) out, err := backticks.Run("/bin/sh", "-c", cmd) if err != nil { trace.Warning("Failure [", out, "] when running command:", cmd) - util.JustExitWith(err) + osutil.ExitErr(err) } } diff --git a/client/go/internal/admin/prog/numactl.go b/client/go/internal/admin/prog/numactl.go index da159529ec0..58bec66e986 100644 --- a/client/go/internal/admin/prog/numactl.go +++ b/client/go/internal/admin/prog/numactl.go @@ -10,7 +10,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) const ( @@ -23,7 +23,7 @@ func (p *Spec) ConfigureNumaCtl() { if p.Getenv(envvars.VESPA_NO_NUMACTL) != "" { return } - backticks := util.BackTicksIgnoreStderr + backticks := osutil.BackTicksIgnoreStderr out, err := backticks.Run(NUMACTL_PROG, "--hardware") trace.Debug("numactl --hardware says:", out) if err != nil { diff --git a/client/go/internal/admin/prog/run.go b/client/go/internal/admin/prog/run.go index 7dafbad1446..31a88610697 100644 --- a/client/go/internal/admin/prog/run.go +++ b/client/go/internal/admin/prog/run.go @@ -5,7 +5,7 @@ package prog import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) func (spec *Spec) Run() error { @@ -22,5 +22,5 @@ func (spec *Spec) Run() error { spec.Setenv(envvars.LD_PRELOAD, spec.vespaMallocPreload) } envv := spec.EffectiveEnv() - return util.Execvpe(prog, args, envv) + return osutil.Execvpe(prog, args, envv) } diff --git a/client/go/internal/admin/prog/valgrind.go b/client/go/internal/admin/prog/valgrind.go index 301146bd444..a8a41b95245 100644 --- a/client/go/internal/admin/prog/valgrind.go +++ b/client/go/internal/admin/prog/valgrind.go @@ -10,7 +10,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -23,7 +23,7 @@ func (p *Spec) ConfigureValgrind() { p.shouldUseCallgrind = false if p.MatchesListEnv(envvars.VESPA_USE_VALGRIND) { trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND) - backticks := util.BackTicksWithStderr + backticks := osutil.BackTicksWithStderr out, err := backticks.Run(VALGRIND_PROG, "--help") if err != nil { trace.Trace("trial run of valgrind fails:", err, "=>", out) diff --git a/client/go/internal/admin/prog/valgrind_test.go b/client/go/internal/admin/prog/valgrind_test.go index 5bc9c1625fe..30dcca0c724 100644 --- a/client/go/internal/admin/prog/valgrind_test.go +++ b/client/go/internal/admin/prog/valgrind_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) var tmpBin string @@ -22,7 +22,7 @@ func useMock(prog, target string) { os.Remove(symlink) err := os.Symlink(mock, symlink) if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } } diff --git a/client/go/internal/admin/vespa-wrapper/configserver/check.go b/client/go/internal/admin/vespa-wrapper/configserver/check.go index 73270271051..a9c1c09348e 100644 --- a/client/go/internal/admin/vespa-wrapper/configserver/check.go +++ b/client/go/internal/admin/vespa-wrapper/configserver/check.go @@ -8,7 +8,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) func checkIsConfigserver(myname string) { @@ -20,5 +20,5 @@ func checkIsConfigserver(myname string) { } } trace.Warning("only these hosts should run a config server:", onlyHosts) - util.JustExitMsg(fmt.Sprintf("this host [%s] should not run a config server", myname)) + osutil.ExitMsg(fmt.Sprintf("this host [%s] should not run a config server", myname)) } diff --git a/client/go/internal/admin/vespa-wrapper/configserver/env.go b/client/go/internal/admin/vespa-wrapper/configserver/env.go index 3b43fc269f8..b4dac91d275 100644 --- a/client/go/internal/admin/vespa-wrapper/configserver/env.go +++ b/client/go/internal/admin/vespa-wrapper/configserver/env.go @@ -8,7 +8,7 @@ import ( "os" "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) func exportSettings(vespaHome string) { @@ -27,5 +27,5 @@ func exportSettings(vespaHome string) { os.Setenv(envvars.STANDALONE_JDISC_APP_LOCATION, app) os.Setenv(envvars.STANDALONE_JDISC_DEPLOYMENT_PROFILE, "configserver") os.Setenv(envvars.MALLOC_ARENA_MAX, "1") - util.OptionallyReduceTimerFrequency() + osutil.OptionallyReduceTimerFrequency() } diff --git a/client/go/internal/admin/vespa-wrapper/configserver/fix_dirs_and_files.go b/client/go/internal/admin/vespa-wrapper/configserver/fix_dirs_and_files.go index 87f5f13d9d0..349366cb928 100644 --- a/client/go/internal/admin/vespa-wrapper/configserver/fix_dirs_and_files.go +++ b/client/go/internal/admin/vespa-wrapper/configserver/fix_dirs_and_files.go @@ -4,13 +4,13 @@ package configserver import ( - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) -func makeFixSpec() util.FixSpec { +func makeFixSpec() osutil.FixSpec { vespaUid, vespaGid := vespa.FindVespaUidAndGid() - return util.FixSpec{ + return osutil.FixSpec{ UserId: vespaUid, GroupId: vespaGid, DirMode: 0755, @@ -18,7 +18,7 @@ func makeFixSpec() util.FixSpec { } } -func fixDirsAndFiles(fixSpec util.FixSpec) { +func fixDirsAndFiles(fixSpec osutil.FixSpec) { fixSpec.FixDir("var/zookeeper") fixSpec.FixDir("var/zookeeper/conf") fixSpec.FixDir("var/zookeeper/version-2") diff --git a/client/go/internal/admin/vespa-wrapper/configserver/logd.go b/client/go/internal/admin/vespa-wrapper/configserver/logd.go index 99fc9aa622a..3d51ce317a2 100644 --- a/client/go/internal/admin/vespa-wrapper/configserver/logd.go +++ b/client/go/internal/admin/vespa-wrapper/configserver/logd.go @@ -8,12 +8,12 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) func maybeStartLogd() { if os.Getenv(envvars.VESPA_CONFIGSERVER_MULTITENANT) == "true" { - backticks := util.BackTicksForwardStderr + backticks := osutil.BackTicksForwardStderr out, err := backticks.Run("libexec/vespa/start-logd") if err != nil { panic(err) diff --git a/client/go/internal/admin/vespa-wrapper/configserver/runserver.go b/client/go/internal/admin/vespa-wrapper/configserver/runserver.go index 25935c814e7..5fdaa5fa9b8 100644 --- a/client/go/internal/admin/vespa-wrapper/configserver/runserver.go +++ b/client/go/internal/admin/vespa-wrapper/configserver/runserver.go @@ -8,7 +8,9 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" + "github.com/vespa-engine/vespa/client/go/internal/list" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) const ( @@ -27,25 +29,25 @@ func (rs *RunServer) PidFile() string { func (rs *RunServer) ProgPath() string { p := fmt.Sprintf("%s/bin64/%s", defaults.VespaHome(), PROG_NAME) - if util.IsExecutableFile(p) { + if ioutil.IsExecutable(p) { return p } p = fmt.Sprintf("%s/bin/%s", defaults.VespaHome(), PROG_NAME) - if util.IsExecutableFile(p) { + if ioutil.IsExecutable(p) { return p } panic(fmt.Errorf("not an executable file: %s", p)) } func (rs *RunServer) WouldRun() bool { - backticks := util.BackTicksForwardStderr + backticks := osutil.BackTicksForwardStderr out, err := backticks.Run(rs.ProgPath(), "-s", rs.ServiceName, "-p", rs.PidFile(), "-W") trace.Trace("output from -W:", out, "error:", err) return err == nil } func (rs *RunServer) Exec(prog string) { - argv := util.ArrayList[string]{ + argv := list.ArrayList[string]{ PROG_NAME, "-s", rs.ServiceName, "-r", "30", @@ -54,6 +56,6 @@ func (rs *RunServer) Exec(prog string) { prog, } argv.AppendAll(rs.Args...) - err := util.Execvp(rs.ProgPath(), argv) - util.JustExitWith(err) + err := osutil.Execvp(rs.ProgPath(), argv) + osutil.ExitErr(err) } diff --git a/client/go/internal/admin/vespa-wrapper/configserver/start.go b/client/go/internal/admin/vespa-wrapper/configserver/start.go index 91064b21849..452ea714100 100644 --- a/client/go/internal/admin/vespa-wrapper/configserver/start.go +++ b/client/go/internal/admin/vespa-wrapper/configserver/start.go @@ -9,7 +9,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -37,7 +37,7 @@ func commonPreChecks() (veHome string) { checkIsConfigserver(veHost) e = os.Chdir(veHome) if e != nil { - util.JustExitWith(e) + osutil.ExitErr(e) } return } @@ -45,8 +45,8 @@ func commonPreChecks() (veHome string) { func JustStartConfigserver() int { vespaHome := commonPreChecks() vespa.CheckCorrectUser() - util.TuneResourceLimits() - util.TuneLogging(SERVICE_NAME, "com.google.api.client.http.HttpTransport", "config=off") + osutil.TuneResourceLimits() + osutil.TuneLogging(SERVICE_NAME, "com.google.api.client.http.HttpTransport", "config=off") exportSettings(vespaHome) removeStaleZkLocks(vespaHome) c := jvm.NewStandaloneContainer(SERVICE_NAME) @@ -76,7 +76,7 @@ func runConfigserverWithRunserver() int { func StartConfigserverEtc() int { vespaHome := commonPreChecks() vespa.RunPreStart() - util.TuneResourceLimits() + osutil.TuneResourceLimits() fixSpec := makeFixSpec() fixDirsAndFiles(fixSpec) exportSettings(vespaHome) diff --git a/client/go/internal/admin/vespa-wrapper/configserver/zk.go b/client/go/internal/admin/vespa-wrapper/configserver/zk.go index 2fee37f0662..c14c8fe8e29 100644 --- a/client/go/internal/admin/vespa-wrapper/configserver/zk.go +++ b/client/go/internal/admin/vespa-wrapper/configserver/zk.go @@ -7,7 +7,7 @@ import ( "fmt" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) const ( @@ -15,7 +15,7 @@ const ( ) func removeStaleZkLocks(vespaHome string) { - backticks := util.BackTicksIgnoreStderr + backticks := osutil.BackTicksIgnoreStderr cmd := fmt.Sprintf("rm -f '%s/%s'*lck", vespaHome, ZOOKEEPER_LOG_FILE_PREFIX) trace.Trace("cleaning locks:", cmd) backticks.Run("/bin/sh", "-c", cmd) diff --git a/client/go/internal/admin/vespa-wrapper/main.go b/client/go/internal/admin/vespa-wrapper/main.go index ad1eee52b63..32c5f909491 100644 --- a/client/go/internal/admin/vespa-wrapper/main.go +++ b/client/go/internal/admin/vespa-wrapper/main.go @@ -17,7 +17,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/vespa-wrapper/services" "github.com/vespa-engine/vespa/client/go/internal/admin/vespa-wrapper/standalone" "github.com/vespa-engine/vespa/client/go/internal/admin/vespa-wrapper/startcbinary" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -99,7 +99,7 @@ func main() { func handleSimplePanic() { if r := recover(); r != nil { - if jee, ok := r.(*util.JustExitError); ok { + if jee, ok := r.(*osutil.ExitError); ok { fmt.Fprintln(os.Stderr, jee) os.Exit(1) } else { diff --git a/client/go/internal/admin/vespa-wrapper/services/configproxy.go b/client/go/internal/admin/vespa-wrapper/services/configproxy.go index e92b05ccc1e..bed6e980cf4 100644 --- a/client/go/internal/admin/vespa-wrapper/services/configproxy.go +++ b/client/go/internal/admin/vespa-wrapper/services/configproxy.go @@ -15,7 +15,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -30,9 +30,9 @@ func JustRunConfigproxy() int { vespa.CheckCorrectUser() configsources := defaults.VespaConfigserverRpcAddrs() if len(configsources) < 1 { - util.JustExitMsg("could not find any configservers") + osutil.ExitMsg("could not find any configservers") } - util.TuneResourceLimits() + osutil.TuneResourceLimits() c := jvm.NewConfigProxyJvm(PROXY_SERVICE_NAME) userargs := os.Getenv(envvars.VESPA_CONFIGPROXY_JVMARGS) c.ConfigureOptions(configsources, userargs) @@ -68,7 +68,7 @@ func startProxyWithRunserver() { func waitForProxyResponse() bool { hname, _ := vespa.FindOurHostname() - backtick := util.BackTicksWithStderr + backtick := osutil.BackTicksWithStderr start := time.Now() fmt.Printf("Waiting for config proxy to start\n") for sleepcount := 0; sleepcount < 1800; sleepcount++ { @@ -135,7 +135,7 @@ func StartConfigproxy() int { } func stopProxyWithRunserver() { - _, err := util.SystemCommand.Run("vespa-runserver", + _, err := osutil.SystemCommand.Run("vespa-runserver", "-s", PROXY_SERVICE_NAME, "-p", CONFIGPROXY_PIDFILE, "-S") if err != nil { diff --git a/client/go/internal/admin/vespa-wrapper/services/env.go b/client/go/internal/admin/vespa-wrapper/services/env.go index 9a7ba40c73e..23cf80213d5 100644 --- a/client/go/internal/admin/vespa-wrapper/services/env.go +++ b/client/go/internal/admin/vespa-wrapper/services/env.go @@ -8,7 +8,7 @@ import ( "os" "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) func exportSettings(vespaHome string) { @@ -21,5 +21,5 @@ func exportSettings(vespaHome string) { os.Setenv(envvars.JAVAVM_LD_PRELOAD, "") os.Setenv(envvars.LD_PRELOAD, "") os.Setenv(envvars.MALLOC_ARENA_MAX, "1") - util.OptionallyReduceTimerFrequency() + osutil.OptionallyReduceTimerFrequency() } diff --git a/client/go/internal/admin/vespa-wrapper/services/prechecks.go b/client/go/internal/admin/vespa-wrapper/services/prechecks.go index bb6bf55e06b..93a53707ba2 100644 --- a/client/go/internal/admin/vespa-wrapper/services/prechecks.go +++ b/client/go/internal/admin/vespa-wrapper/services/prechecks.go @@ -8,7 +8,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -31,7 +31,7 @@ func commonPreChecks() (veHome, veHost string) { } err = os.Chdir(veHome) if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } return } diff --git a/client/go/internal/admin/vespa-wrapper/services/sentinel.go b/client/go/internal/admin/vespa-wrapper/services/sentinel.go index 7694c930731..e489bf70ded 100644 --- a/client/go/internal/admin/vespa-wrapper/services/sentinel.go +++ b/client/go/internal/admin/vespa-wrapper/services/sentinel.go @@ -13,7 +13,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -48,7 +48,7 @@ func startSentinelWithRunserver() { } func waitForSentinelPid() bool { - backtick := util.BackTicksWithStderr + backtick := osutil.BackTicksWithStderr start := time.Now() for sleepcount := 0; sleepcount < 1000; sleepcount++ { time.Sleep(10 * time.Millisecond) @@ -84,7 +84,7 @@ func StartConfigSentinel() int { } func stopSentinelWithRunserver() { - _, err := util.SystemCommand.Run("vespa-runserver", + _, err := osutil.SystemCommand.Run("vespa-runserver", "-s", SENTINEL_SERVICE_NAME, "-p", SENTINEL_PIDFILE, "-S") if err != nil { diff --git a/client/go/internal/admin/vespa-wrapper/services/start.go b/client/go/internal/admin/vespa-wrapper/services/start.go index d2c2fb6f5ba..de658264669 100644 --- a/client/go/internal/admin/vespa-wrapper/services/start.go +++ b/client/go/internal/admin/vespa-wrapper/services/start.go @@ -10,7 +10,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -33,14 +33,14 @@ func StartServices() int { } func checkjava() { - backticks := util.BackTicksWithStderr + backticks := osutil.BackTicksWithStderr out, err := backticks.Run("java", "-version") if err != nil { trace.Warning("cannot run 'java -version'") - util.JustExitWith(err) + osutil.ExitErr(err) } if !strings.Contains(out, "64-Bit Server VM") { - util.JustExitWith(fmt.Errorf("java must invoke the 64-bit Java VM, but -version says:\n%s\n", out)) + osutil.ExitErr(fmt.Errorf("java must invoke the 64-bit Java VM, but -version says:\n%s\n", out)) } } @@ -53,7 +53,7 @@ func VespaStartServices() int { trace.Debug("common prechecks ok, running in", home, "on", host) vespa.RunPreStart() trace.Debug("prestart ok") - util.TuneResourceLimits() + osutil.TuneResourceLimits() increase_vm_max_map_count() trace.Debug("resource limits ok") checkjava() @@ -64,7 +64,7 @@ func VespaStartServices() int { drop_caches() err := vespa.MaybeSwitchUser("start-services") if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } return StartServices() } diff --git a/client/go/internal/admin/vespa-wrapper/services/stop.go b/client/go/internal/admin/vespa-wrapper/services/stop.go index 54d557847ce..c4c4787c83a 100644 --- a/client/go/internal/admin/vespa-wrapper/services/stop.go +++ b/client/go/internal/admin/vespa-wrapper/services/stop.go @@ -8,7 +8,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -21,11 +21,11 @@ func VespaStopServices() int { } err := vespa.LoadDefaultEnv() if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } err = vespa.MaybeSwitchUser("vespa-stop-services") if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } vespa.CheckCorrectUser() trace.Debug("running as correct user") diff --git a/client/go/internal/admin/vespa-wrapper/standalone/start.go b/client/go/internal/admin/vespa-wrapper/standalone/start.go index add29d37671..a3703ce930c 100644 --- a/client/go/internal/admin/vespa-wrapper/standalone/start.go +++ b/client/go/internal/admin/vespa-wrapper/standalone/start.go @@ -9,7 +9,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/jvm" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -23,20 +23,20 @@ func commonPreChecks() { veHome := vespa.FindAndVerifyVespaHome() err := os.Chdir(veHome) if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } err = vespa.LoadDefaultEnv() if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } } func StartStandaloneContainer(extraArgs []string) int { commonPreChecks() - util.TuneResourceLimits() + osutil.TuneResourceLimits() serviceName := os.Getenv("VESPA_SERVICE_NAME") if serviceName == "" { - util.JustExitMsg("Missing service name, ensure VESPA_SERVICE_NAME is set in the environment") + osutil.ExitMsg("Missing service name, ensure VESPA_SERVICE_NAME is set in the environment") } c := jvm.NewStandaloneContainer(serviceName) jvmOpts := c.JvmOptions() diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/cmd.go b/client/go/internal/admin/vespa-wrapper/startcbinary/cmd.go index 8902aef80df..a35f0c5b820 100644 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/cmd.go +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/cmd.go @@ -9,7 +9,8 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" + "github.com/vespa-engine/vespa/client/go/internal/osutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -22,7 +23,7 @@ func Run(args []string) int { spec := NewProgSpec(args) err := vespa.LoadDefaultEnv() if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } hostname, err := vespa.FindOurHostname() if err != nil { @@ -34,12 +35,12 @@ func Run(args []string) int { func IsCandidate(program string) bool { binary := program + "-bin" if strings.Contains(binary, "/") { - return util.IsRegularFile(binary) + return ioutil.IsFile(binary) } else { path := strings.Split(os.Getenv(envvars.PATH), ":") for _, dir := range path { fn := dir + "/" + binary - if util.IsRegularFile(fn) { + if ioutil.IsFile(fn) { return true } } diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go b/client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go index 898e4558152..06c20021ff6 100644 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go @@ -4,10 +4,10 @@ package startcbinary import ( - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) func configureTuning() { - util.OptionallyReduceTimerFrequency() - util.TuneResourceLimits() + osutil.OptionallyReduceTimerFrequency() + osutil.TuneResourceLimits() } diff --git a/client/go/internal/cli/auth/auth0/auth0.go b/client/go/internal/cli/auth/auth0/auth0.go index 7fae6e78b61..9466e9865b5 100644 --- a/client/go/internal/cli/auth/auth0/auth0.go +++ b/client/go/internal/cli/auth/auth0/auth0.go @@ -14,7 +14,7 @@ import ( "time" "github.com/vespa-engine/vespa/client/go/internal/cli/auth" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" ) const ( @@ -31,7 +31,7 @@ type Credentials struct { // Client is a client for the Auth0 service. type Client struct { - httpClient util.HTTPClient + httpClient httputil.Client Authenticator *auth.Authenticator // TODO: Make this private options Options provider auth0Provider @@ -80,7 +80,7 @@ func cancelOnInterrupt() context.Context { // NewClient constructs a new Auth0 client, storing configuration in the given configPath. The client will be configured for // use in the given Vespa system. -func NewClient(httpClient util.HTTPClient, options Options) (*Client, error) { +func NewClient(httpClient httputil.Client, options Options) (*Client, error) { a := Client{} a.httpClient = httpClient a.options = options diff --git a/client/go/internal/cli/auth/zts/zts.go b/client/go/internal/cli/auth/zts/zts.go index b60aa363e70..1c31ba05dd3 100644 --- a/client/go/internal/cli/auth/zts/zts.go +++ b/client/go/internal/cli/auth/zts/zts.go @@ -11,7 +11,7 @@ import ( "sync" "time" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" ) const ( @@ -21,7 +21,7 @@ const ( // Client is a client for Athenz ZTS, an authentication token service. type Client struct { - client util.HTTPClient + client httputil.Client tokenURL *url.URL domain string now func() time.Time @@ -38,7 +38,7 @@ type Token struct { func (t *Token) isExpired(now time.Time) bool { return t.ExpiresAt.Sub(now) < expirySlack } // NewClient creates a new client for an Athenz ZTS service located at serviceURL. -func NewClient(client util.HTTPClient, domain, serviceURL string) (*Client, error) { +func NewClient(client httputil.Client, domain, serviceURL string) (*Client, error) { tokenURL, err := url.Parse(serviceURL) if err != nil { return nil, err diff --git a/client/go/internal/cli/cmd/api_key.go b/client/go/internal/cli/cmd/api_key.go index 133c9db0d3b..d882c527516 100644 --- a/client/go/internal/cli/cmd/api_key.go +++ b/client/go/internal/cli/cmd/api_key.go @@ -10,7 +10,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -71,7 +71,7 @@ func doApiKey(cli *CLI, overwriteKey bool, args []string) error { return err } apiKeyFile := cli.config.apiKeyPath(app.Tenant) - if util.PathExists(apiKeyFile) && !overwriteKey { + if ioutil.Exists(apiKeyFile) && !overwriteKey { err := fmt.Errorf("refusing to overwrite %s", apiKeyFile) cli.printErr(err, "Use -f to overwrite it") printPublicKey(system, apiKeyFile, app.Tenant) diff --git a/client/go/internal/cli/cmd/cert.go b/client/go/internal/cli/cmd/cert.go index 6aa99a01902..3e18fafb815 100644 --- a/client/go/internal/cli/cmd/cert.go +++ b/client/go/internal/cli/cmd/cert.go @@ -10,7 +10,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -122,10 +122,10 @@ func doCert(cli *CLI, overwriteCertificate, skipApplicationPackage bool, args [] if !overwriteCertificate { hint := "Use -f flag to force overwriting" - if util.PathExists(privateKeyFile.path) { + if ioutil.Exists(privateKeyFile.path) { return errHint(fmt.Errorf("private key %s already exists", color.CyanString(privateKeyFile.path)), hint) } - if util.PathExists(certificateFile.path) { + if ioutil.Exists(certificateFile.path) { return errHint(fmt.Errorf("certificate %s already exists", color.CyanString(certificateFile.path)), hint) } } @@ -213,7 +213,7 @@ func copyCertificate(cli *CLI, target vespa.Target, pkg vespa.ApplicationPackage if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { return fmt.Errorf("could not create security directory: %w", err) } - err = util.AtomicWriteFile(dstPath, data) + err = ioutil.AtomicWriteFile(dstPath, data) if err == nil { cli.printSuccess("Copied certificate from ", tlsOptions.CertificateFile, " to ", dstPath) } diff --git a/client/go/internal/cli/cmd/clone_list.go b/client/go/internal/cli/cmd/clone_list.go index 40656841276..c6a99533c03 100644 --- a/client/go/internal/cli/cmd/clone_list.go +++ b/client/go/internal/cli/cmd/clone_list.go @@ -7,14 +7,14 @@ import ( "sort" "time" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" ) -func listSampleApps(client util.HTTPClient) ([]string, error) { +func listSampleApps(client httputil.Client) ([]string, error) { return listSampleAppsAt("https://api.github.com/repos/vespa-engine/sample-apps/contents/", client) } -func listSampleAppsAt(url string, client util.HTTPClient) ([]string, error) { +func listSampleAppsAt(url string, client httputil.Client) ([]string, error) { rfs, err := getRepositoryFiles(url, client) if err != nil { return nil, err @@ -36,7 +36,7 @@ func listSampleAppsAt(url string, client util.HTTPClient) ([]string, error) { return apps, nil } -func getRepositoryFiles(url string, client util.HTTPClient) ([]repositoryFile, error) { +func getRepositoryFiles(url string, client httputil.Client) ([]repositoryFile, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err diff --git a/client/go/internal/cli/cmd/clone_test.go b/client/go/internal/cli/cmd/clone_test.go index 331845b3883..e783f75d9d4 100644 --- a/client/go/internal/cli/cmd/clone_test.go +++ b/client/go/internal/cli/cmd/clone_test.go @@ -13,8 +13,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/mock" - "github.com/vespa-engine/vespa/client/go/internal/util" ) func TestClone(t *testing.T) { @@ -101,9 +101,9 @@ func TestClone(t *testing.T) { func assertFiles(t *testing.T, app string) { t.Helper() - assert.True(t, util.PathExists(filepath.Join(app, "README.md"))) - assert.True(t, util.PathExists(filepath.Join(app, "src", "main", "application"))) - assert.True(t, util.IsDirectory(filepath.Join(app, "src", "main", "application"))) + assert.True(t, ioutil.Exists(filepath.Join(app, "README.md"))) + assert.True(t, ioutil.Exists(filepath.Join(app, "src", "main", "application"))) + assert.True(t, ioutil.IsDir(filepath.Join(app, "src", "main", "application"))) servicesStat, err := os.Stat(filepath.Join(app, "src", "main", "application", "services.xml")) require.Nil(t, err) diff --git a/client/go/internal/cli/cmd/document.go b/client/go/internal/cli/cmd/document.go index 6892956880b..0393a9b2595 100644 --- a/client/go/internal/cli/cmd/document.go +++ b/client/go/internal/cli/cmd/document.go @@ -15,7 +15,8 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" "github.com/vespa-engine/vespa/client/go/internal/vespa/document" ) @@ -39,7 +40,7 @@ func documentClient(cli *CLI, timeoutSecs, waitSecs int, printCurl bool) (*docum Timeout: time.Duration(timeoutSecs) * time.Second, BaseURL: docService.BaseURL, NowFunc: time.Now, - }, []util.HTTPClient{docService}) + }, []httputil.Client{docService}) if err != nil { return nil, nil, err } @@ -103,22 +104,22 @@ func readDocument(id string, timeoutSecs, waitSecs int, printCurl bool, cli *CLI return printResult(cli, operationResult(true, document.Document{Id: docId}, service, result), true) } -func operationResult(read bool, doc document.Document, service *vespa.Service, result document.Result) util.OperationResult { +func operationResult(read bool, doc document.Document, service *vespa.Service, result document.Result) OperationResult { if result.Err != nil { - return util.Failure(result.Err.Error()) + return Failure(result.Err.Error()) } bodyReader := bytes.NewReader(result.Body) if result.HTTPStatus == 200 { if read { - return util.SuccessWithPayload("Read "+doc.Id.String(), util.ReaderToJSON(bodyReader)) + return SuccessWithPayload("Read "+doc.Id.String(), ioutil.ReaderToJSON(bodyReader)) } else { - return util.Success(doc.Operation.String() + " " + doc.Id.String()) + return Success(doc.Operation.String() + " " + doc.Id.String()) } } if result.HTTPStatus/100 == 4 { - return util.FailureWithPayload("Invalid document operation: Status "+strconv.Itoa(result.HTTPStatus), util.ReaderToJSON(bodyReader)) + return FailureWithPayload("Invalid document operation: Status "+strconv.Itoa(result.HTTPStatus), ioutil.ReaderToJSON(bodyReader)) } - return util.FailureWithPayload(service.Description()+" at "+service.BaseURL+": Status "+strconv.Itoa(result.HTTPStatus), util.ReaderToJSON(bodyReader)) + return FailureWithPayload(service.Description()+" at "+service.BaseURL+": Status "+strconv.Itoa(result.HTTPStatus), ioutil.ReaderToJSON(bodyReader)) } func newDocumentCmd(cli *CLI) *cobra.Command { @@ -269,7 +270,7 @@ func documentService(cli *CLI, waitSecs int) (*vespa.Service, error) { return waiter.Service(target, cli.config.cluster()) } -func printResult(cli *CLI, result util.OperationResult, payloadOnlyOnSuccess bool) error { +func printResult(cli *CLI, result OperationResult, payloadOnlyOnSuccess bool) error { out := cli.Stdout if !result.Success { out = cli.Stderr diff --git a/client/go/internal/cli/cmd/document_test.go b/client/go/internal/cli/cmd/document_test.go index 0b8d5a50615..3cfc66fdad4 100644 --- a/client/go/internal/cli/cmd/document_test.go +++ b/client/go/internal/cli/cmd/document_test.go @@ -12,8 +12,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/mock" - "github.com/vespa-engine/vespa/client/go/internal/util" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -148,7 +148,7 @@ func assertDocumentSend(args []string, expectedOperation string, expectedMethod Fields json.RawMessage `json:"fields"` } assert.Nil(t, json.Unmarshal(data, &expectedPayload)) - assert.Equal(t, `{"fields":`+string(expectedPayload.Fields)+"}", util.ReaderToString(client.LastRequest.Body)) + assert.Equal(t, `{"fields":`+string(expectedPayload.Fields)+"}", ioutil.ReaderToString(client.LastRequest.Body)) } else { assert.Nil(t, client.LastRequest.Body) } diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go index 89e13a4673c..69b847547a9 100644 --- a/client/go/internal/cli/cmd/feed.go +++ b/client/go/internal/cli/cmd/feed.go @@ -12,7 +12,7 @@ import ( "time" "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" "github.com/vespa-engine/vespa/client/go/internal/vespa/document" ) @@ -124,7 +124,7 @@ $ cat docs.jsonl | vespa feed -`, return cmd } -func createServices(n int, timeout time.Duration, waitSecs int, cli *CLI) ([]util.HTTPClient, string, error) { +func createServices(n int, timeout time.Duration, waitSecs int, cli *CLI) ([]httputil.Client, string, error) { if n < 1 { return nil, "", fmt.Errorf("need at least one client") } @@ -132,7 +132,7 @@ func createServices(n int, timeout time.Duration, waitSecs int, cli *CLI) ([]uti if err != nil { return nil, "", err } - services := make([]util.HTTPClient, 0, n) + services := make([]httputil.Client, 0, n) baseURL := "" waiter := cli.waiter(time.Duration(waitSecs) * time.Second) for i := 0; i < n; i++ { @@ -144,7 +144,7 @@ func createServices(n int, timeout time.Duration, waitSecs int, cli *CLI) ([]uti // Create a separate HTTP client for each service client := cli.httpClientFactory(timeout) // Feeding should always use HTTP/2 - util.ForceHTTP2(client, service.TLSOptions.KeyPair, service.TLSOptions.CACertificate, service.TLSOptions.TrustAll) + httputil.ForceHTTP2(client, service.TLSOptions.KeyPair, service.TLSOptions.CACertificate, service.TLSOptions.TrustAll) service.SetClient(client) services = append(services, service) } diff --git a/client/go/internal/cli/cmd/man_test.go b/client/go/internal/cli/cmd/man_test.go index ad05efcb2a3..ae434624ac7 100644 --- a/client/go/internal/cli/cmd/man_test.go +++ b/client/go/internal/cli/cmd/man_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" ) func TestMan(t *testing.T) { @@ -15,5 +15,5 @@ func TestMan(t *testing.T) { cli, stdout, _ := newTestCLI(t) assert.Nil(t, cli.Run("man", tmpDir)) assert.Equal(t, fmt.Sprintf("Success: Man pages written to %s\n", tmpDir), stdout.String()) - assert.True(t, util.PathExists(filepath.Join(tmpDir, "vespa.1"))) + assert.True(t, ioutil.Exists(filepath.Join(tmpDir, "vespa.1"))) } diff --git a/client/go/internal/cli/cmd/prod.go b/client/go/internal/cli/cmd/prod.go index ddf2995126a..0912ca31e25 100644 --- a/client/go/internal/cli/cmd/prod.go +++ b/client/go/internal/cli/cmd/prod.go @@ -15,7 +15,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" "github.com/vespa-engine/vespa/client/go/internal/vespa/xml" ) @@ -187,7 +187,7 @@ $ vespa prod deploy`, func writeWithBackup(stdout io.Writer, pkg vespa.ApplicationPackage, filename, contents string) error { dst := filepath.Join(pkg.Path, filename) - if util.PathExists(dst) { + if ioutil.Exists(dst) { data, err := os.ReadFile(dst) if err != nil { return err @@ -199,7 +199,7 @@ func writeWithBackup(stdout io.Writer, pkg vespa.ApplicationPackage, filename, c renamed := false for i := 1; i <= 1000; i++ { bak := fmt.Sprintf("%s.%d.bak", dst, i) - if !util.PathExists(bak) { + if !ioutil.Exists(bak) { fmt.Fprintf(stdout, "Backing up existing %s to %s\n", color.YellowString(filename), color.YellowString(bak)) if err := os.Rename(dst, bak); err != nil { return err diff --git a/client/go/internal/cli/cmd/prod_test.go b/client/go/internal/cli/cmd/prod_test.go index 7f2836125d8..6d8a50124ac 100644 --- a/client/go/internal/cli/cmd/prod_test.go +++ b/client/go/internal/cli/cmd/prod_test.go @@ -10,8 +10,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/mock" - "github.com/vespa-engine/vespa/client/go/internal/util" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -76,8 +76,8 @@ func TestProdInit(t *testing.T) { assert.Contains(t, servicesXML, contentFragment) // Backups are created - assert.True(t, util.PathExists(deploymentPath+".1.bak")) - assert.True(t, util.PathExists(servicesPath+".1.bak")) + assert.True(t, ioutil.Exists(deploymentPath+".1.bak")) + assert.True(t, ioutil.Exists(servicesPath+".1.bak")) } func readFileString(t *testing.T, filename string) string { diff --git a/client/go/internal/cli/cmd/query.go b/client/go/internal/cli/cmd/query.go index bf2272ca981..3e5a60a15df 100644 --- a/client/go/internal/cli/cmd/query.go +++ b/client/go/internal/cli/cmd/query.go @@ -16,7 +16,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/internal/curl" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -98,11 +98,11 @@ func query(cli *CLI, arguments []string, timeoutSecs, waitSecs int, curl bool) e defer response.Body.Close() if response.StatusCode == 200 { - log.Print(util.ReaderToJSON(response.Body)) + log.Print(ioutil.ReaderToJSON(response.Body)) } else if response.StatusCode/100 == 4 { - return fmt.Errorf("invalid query: %s\n%s", response.Status, util.ReaderToJSON(response.Body)) + return fmt.Errorf("invalid query: %s\n%s", response.Status, ioutil.ReaderToJSON(response.Body)) } else { - return fmt.Errorf("%s from container at %s\n%s", response.Status, color.CyanString(url.Host), util.ReaderToJSON(response.Body)) + return fmt.Errorf("%s from container at %s\n%s", response.Status, color.CyanString(url.Host), ioutil.ReaderToJSON(response.Body)) } return nil } diff --git a/client/go/internal/util/operation_result.go b/client/go/internal/cli/cmd/result.go index 7dc60f92e1d..65b7ec4ca63 100644 --- a/client/go/internal/util/operation_result.go +++ b/client/go/internal/cli/cmd/result.go @@ -1,8 +1,4 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// A struct containing the result of an operation -// Author: bratseth - -package util +package cmd type OperationResult struct { Success bool diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index 1f324658b69..383ce7dd28d 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/briandowns/spinner" "github.com/fatih/color" "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" @@ -20,7 +21,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/build" "github.com/vespa-engine/vespa/client/go/internal/cli/auth/auth0" "github.com/vespa-engine/vespa/client/go/internal/cli/auth/zts" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" "github.com/vespa-engine/vespa/client/go/internal/version" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -58,8 +59,8 @@ type CLI struct { config *Config version version.Version - httpClient util.HTTPClient - httpClientFactory func(timeout time.Duration) util.HTTPClient + httpClient httputil.Client + httpClientFactory func(timeout time.Duration) httputil.Client auth0Factory auth0Factory ztsFactory ztsFactory } @@ -102,9 +103,33 @@ func (c *execSubprocess) Run(name string, args ...string) ([]byte, error) { return exec.Command(name, args...).Output() } -type auth0Factory func(httpClient util.HTTPClient, options auth0.Options) (vespa.Authenticator, error) +type auth0Factory func(httpClient httputil.Client, options auth0.Options) (vespa.Authenticator, error) -type ztsFactory func(httpClient util.HTTPClient, domain, url string) (vespa.Authenticator, error) +type ztsFactory func(httpClient httputil.Client, domain, url string) (vespa.Authenticator, error) + +// newSpinner writes message to writer w and executes function fn. While fn is running a spinning animation will be +// displayed after message. +func newSpinner(w io.Writer, message string, fn func() error) error { + s := spinner.New(spinner.CharSets[11], 100*time.Millisecond, spinner.WithWriter(w)) + // Cursor is hidden by default. Hiding cursor requires Stop() to be called to restore cursor (i.e. if the process is + // interrupted), however we don't want to bother with a signal handler just for this + s.HideCursor = false + if err := s.Color("blue", "bold"); err != nil { + return err + } + if !strings.HasSuffix(message, " ") { + message += " " + } + s.Prefix = message + s.FinalMSG = "\r" + message + "done\n" + s.Start() + err := fn() + if err != nil { + s.FinalMSG = "\r" + message + "failed\n" + } + s.Stop() + return err +} // New creates the Vespa CLI, writing output to stdout and stderr, and reading environment variables from environment. func New(stdout, stderr io.Writer, environment []string) (*CLI, error) { @@ -136,7 +161,7 @@ For detailed description of flags and configuration, see 'vespa help config'. if err != nil { return nil, err } - httpClientFactory := util.CreateClient + httpClientFactory := httputil.NewClient cli := CLI{ Environment: env, Stdin: os.Stdin, @@ -152,10 +177,10 @@ For detailed description of flags and configuration, see 'vespa help config'. httpClient: httpClientFactory(time.Second * 10), httpClientFactory: httpClientFactory, - auth0Factory: func(httpClient util.HTTPClient, options auth0.Options) (vespa.Authenticator, error) { + auth0Factory: func(httpClient httputil.Client, options auth0.Options) (vespa.Authenticator, error) { return auth0.NewClient(httpClient, options) }, - ztsFactory: func(httpClient util.HTTPClient, domain, url string) (vespa.Authenticator, error) { + ztsFactory: func(httpClient httputil.Client, domain, url string) (vespa.Authenticator, error) { return zts.NewClient(httpClient, domain, url) }, } @@ -239,7 +264,7 @@ func (c *CLI) configureSpinner() { return fn() } } else { - c.spinner = util.Spinner + c.spinner = newSpinner } } diff --git a/client/go/internal/cli/cmd/test.go b/client/go/internal/cli/cmd/test.go index 376611767d9..3bc78fc91c8 100644 --- a/client/go/internal/cli/cmd/test.go +++ b/client/go/internal/cli/cmd/test.go @@ -21,7 +21,8 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -277,7 +278,7 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin var response *http.Response if externalEndpoint { - util.ConfigureTLS(context.cli.httpClient, []tls.Certificate{}, nil, false) + httputil.ConfigureTLS(context.cli.httpClient, []tls.Certificate{}, nil, false) response, err = context.cli.httpClient.Do(request, 60*time.Second) } else { response, err = service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout @@ -294,7 +295,7 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin color.RedString(strconv.Itoa(response.StatusCode)), color.CyanString(method), color.CyanString(requestUrl.String()), - util.ReaderToJSON(response.Body)), nil + ioutil.ReaderToJSON(response.Body)), nil } if responseBodySpec == nil { diff --git a/client/go/internal/cli/cmd/test_test.go b/client/go/internal/cli/cmd/test_test.go index 1888db017d4..728e8c29691 100644 --- a/client/go/internal/cli/cmd/test_test.go +++ b/client/go/internal/cli/cmd/test_test.go @@ -15,8 +15,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/mock" - "github.com/vespa-engine/vespa/client/go/internal/util" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -205,6 +205,6 @@ func assertRequests(requests []*http.Request, client *mock.HTTPClient, t *testin if actualBody == nil { actualBody = io.NopCloser(strings.NewReader("")) } - assert.Equal(t, util.ReaderToJSON(want.Body), util.ReaderToJSON(actualBody)) + assert.Equal(t, ioutil.ReaderToJSON(want.Body), ioutil.ReaderToJSON(actualBody)) } } diff --git a/client/go/internal/cli/cmd/testutil_test.go b/client/go/internal/cli/cmd/testutil_test.go index 89f40035f6a..dbeb281a4a8 100644 --- a/client/go/internal/cli/cmd/testutil_test.go +++ b/client/go/internal/cli/cmd/testutil_test.go @@ -11,8 +11,8 @@ import ( "time" "github.com/vespa-engine/vespa/client/go/internal/cli/auth/auth0" + "github.com/vespa-engine/vespa/client/go/internal/httputil" "github.com/vespa-engine/vespa/client/go/internal/mock" - "github.com/vespa-engine/vespa/client/go/internal/util" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -31,13 +31,13 @@ func newTestCLI(t *testing.T, envVars ...string) (*CLI, *bytes.Buffer, *bytes.Bu t.Fatal(err) } httpClient := &mock.HTTPClient{} - cli.httpClientFactory = func(timeout time.Duration) util.HTTPClient { return httpClient } + cli.httpClientFactory = func(timeout time.Duration) httputil.Client { return httpClient } cli.httpClient = httpClient cli.exec = &mock.Exec{} - cli.auth0Factory = func(httpClient util.HTTPClient, options auth0.Options) (vespa.Authenticator, error) { + cli.auth0Factory = func(httpClient httputil.Client, options auth0.Options) (vespa.Authenticator, error) { return &mockAuthenticator{}, nil } - cli.ztsFactory = func(httpClient util.HTTPClient, domain, url string) (vespa.Authenticator, error) { + cli.ztsFactory = func(httpClient httputil.Client, domain, url string) (vespa.Authenticator, error) { return &mockAuthenticator{}, nil } return cli, &stdout, &stderr diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go index f6e2f64e534..963833337c2 100644 --- a/client/go/internal/cli/cmd/visit.go +++ b/client/go/internal/cli/cmd/visit.go @@ -16,7 +16,7 @@ import ( "time" "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -157,26 +157,26 @@ func getEpoch(timeStamp string) (int64, error) { return t, nil } -func checkArguments(vArgs visitArgs) (res util.OperationResult) { +func checkArguments(vArgs visitArgs) (res OperationResult) { if vArgs.slices > 0 || vArgs.sliceId > -1 { if !(vArgs.slices > 0 && vArgs.sliceId > -1) { - return util.Failure("Both 'slices' and 'slice-id' must be set") + return Failure("Both 'slices' and 'slice-id' must be set") } if vArgs.sliceId >= vArgs.slices { - return util.Failure("The 'slice-id' must be in range [0, slices)") + return Failure("The 'slice-id' must be in range [0, slices)") } } // to and from will support RFC3339 format soon, add more validation then if vArgs.from != "" { _, err := getEpoch(vArgs.from) if err != nil { - return util.Failure("Invalid 'from' argument: '" + vArgs.from + "': " + err.Error()) + return Failure("Invalid 'from' argument: '" + vArgs.from + "': " + err.Error()) } } if vArgs.to != "" { _, err := getEpoch(vArgs.to) if err != nil { - return util.Failure("Invalid 'to' argument: '" + vArgs.to + "': " + err.Error()) + return Failure("Invalid 'to' argument: '" + vArgs.to + "': " + err.Error()) } } for _, b := range vArgs.bucketSpaces { @@ -186,10 +186,10 @@ func checkArguments(vArgs visitArgs) (res util.OperationResult) { "global": // Do nothing default: - return util.Failure("Invalid 'bucket-space' argument '" + b + "', must be 'default' or 'global'") + return Failure("Invalid 'bucket-space' argument '" + b + "', must be 'default' or 'global'") } } - return util.Success("") + return Success("") } type HandlersInfo struct { @@ -208,11 +208,11 @@ func parseHandlersOutput(r io.Reader) (*HandlersInfo, error) { return &handlersInfo, err } -func probeHandler(service *vespa.Service, cli *CLI) (res util.OperationResult) { +func probeHandler(service *vespa.Service, cli *CLI) (res OperationResult) { urlPath := service.BaseURL + "/" url, urlParseError := url.Parse(urlPath) if urlParseError != nil { - return util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) + return Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) } request := &http.Request{ URL: url, @@ -221,20 +221,20 @@ func probeHandler(service *vespa.Service, cli *CLI) (res util.OperationResult) { timeout := time.Duration(90) * time.Second response, err := service.Do(request, timeout) if err != nil { - return util.Failure("Request failed: " + err.Error()) + return Failure("Request failed: " + err.Error()) } defer response.Body.Close() if response.StatusCode == 200 { handlersInfo, err := parseHandlersOutput(response.Body) if err != nil || len(handlersInfo.Handlers) == 0 { cli.printWarning("Could not parse JSON response from"+urlPath, err.Error()) - return util.Failure("Bad endpoint") + return Failure("Bad endpoint") } for _, h := range handlersInfo.Handlers { if strings.HasSuffix(h.HandlerClass, "DocumentV1ApiHandler") { for _, binding := range h.ServerBindings { if strings.Contains(binding, "/document/v1/") { - return util.Success("handler OK") + return Success("handler OK") } } w := fmt.Sprintf("expected /document/v1/ binding, but got: %v", h.ServerBindings) @@ -242,13 +242,13 @@ func probeHandler(service *vespa.Service, cli *CLI) (res util.OperationResult) { } } cli.printWarning("Missing /document/v1/ API; add <document-api /> to the container cluster declaration in services.xml") - return util.Failure("Missing /document/v1 API") + return Failure("Missing /document/v1 API") } else { - return util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) + return FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, ioutil.ReaderToJSON(response.Body)) } } -func visitClusters(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) { +func visitClusters(vArgs *visitArgs, service *vespa.Service) (res OperationResult) { clusters := []string{ vArgs.contentCluster, } @@ -294,7 +294,7 @@ func probeVisit(vArgs *visitArgs, service *vespa.Service) []string { return clusters } -func runVisit(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) { +func runVisit(vArgs *visitArgs, service *vespa.Service) (res OperationResult) { vArgs.debugPrint(fmt.Sprintf("trying to visit: '%s'", vArgs.contentCluster)) var totalDocuments int = 0 var continuationToken string @@ -340,7 +340,7 @@ func quoteArgForUrl(arg string) string { return buf.String() } -func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*VespaVisitOutput, util.OperationResult) { +func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*VespaVisitOutput, OperationResult) { urlPath := service.BaseURL + "/document/v1/?cluster=" + quoteArgForUrl(vArgs.contentCluster) if vArgs.fieldSet != "" { urlPath = urlPath + "&fieldSet=" + quoteArgForUrl(vArgs.fieldSet) @@ -370,7 +370,7 @@ func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*V } url, urlParseError := url.Parse(urlPath) if urlParseError != nil { - return nil, util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) + return nil, Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) } request := &http.Request{ URL: url, @@ -379,7 +379,7 @@ func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*V timeout := time.Duration(900) * time.Second response, err := service.Do(request, timeout) if err != nil { - return nil, util.Failure("Request failed: " + err.Error()) + return nil, Failure("Request failed: " + err.Error()) } defer response.Body.Close() vvo, err := parseVisitOutput(response.Body) @@ -390,16 +390,16 @@ func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*V vArgs.cli.printWarning(fmt.Sprintf("Inconsistent contents from: %v", url)) vArgs.cli.printWarning(fmt.Sprintf("claimed count: %d", vvo.DocumentCount)) vArgs.cli.printWarning(fmt.Sprintf("document blobs: %d", len(vvo.Documents))) - return nil, util.Failure("Inconsistent contents from document API") + return nil, Failure("Inconsistent contents from document API") } - return vvo, util.Success("visited " + vArgs.contentCluster) + return vvo, Success("visited " + vArgs.contentCluster) } else { - return nil, util.Failure("error reading response: " + err.Error()) + return nil, Failure("error reading response: " + err.Error()) } } else if response.StatusCode/100 == 4 { - return vvo, util.FailureWithPayload("Invalid document operation: "+response.Status, util.ReaderToJSON(response.Body)) + return vvo, FailureWithPayload("Invalid document operation: "+response.Status, ioutil.ReaderToJSON(response.Body)) } else { - return vvo, util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) + return vvo, FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, ioutil.ReaderToJSON(response.Body)) } } diff --git a/client/go/internal/util/http.go b/client/go/internal/httputil/httputil.go index a7a9de5b8e4..e1e27de5523 100644 --- a/client/go/internal/util/http.go +++ b/client/go/internal/httputil/httputil.go @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package util +package httputil import ( "context" @@ -14,15 +14,16 @@ import ( "golang.org/x/net/http2" ) -type HTTPClient interface { +// Client represents a HTTP client usable by the Vespa CLI. +type Client interface { Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) } -type defaultHTTPClient struct { +type defaultClient struct { client *http.Client } -func (c *defaultHTTPClient) Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) { +func (c *defaultClient) Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) { if c.client.Timeout != timeout { // Set wanted timeout c.client.Timeout = timeout } @@ -33,8 +34,10 @@ func (c *defaultHTTPClient) Do(request *http.Request, timeout time.Duration) (re return c.client.Do(request) } -func ConfigureTLS(client HTTPClient, certificates []tls.Certificate, caCertificate []byte, trustAll bool) { - c, ok := client.(*defaultHTTPClient) +// ConfigureTLS configures the given client with given certificates and caCertificate. If trustAll is true, the client +// will skip verification of the certificate chain. +func ConfigureTLS(client Client, certificates []tls.Certificate, caCertificate []byte, trustAll bool) { + c, ok := client.(*defaultClient) if !ok { return } @@ -60,8 +63,10 @@ func ConfigureTLS(client HTTPClient, certificates []tls.Certificate, caCertifica } } -func ForceHTTP2(client HTTPClient, certificates []tls.Certificate, caCertificate []byte, trustAll bool) { - c, ok := client.(*defaultHTTPClient) +// ForceHTTP2 configures the given client exclusively with a HTTP/2 transport. The other options are passed to +// ConfigureTLS. If certificates is nil, the client will be configured with H2C (HTTP/2 over clear-text). +func ForceHTTP2(client Client, certificates []tls.Certificate, caCertificate []byte, trustAll bool) { + c, ok := client.(*defaultClient) if !ok { return } @@ -85,8 +90,9 @@ func ForceHTTP2(client HTTPClient, certificates []tls.Certificate, caCertificate ConfigureTLS(client, certificates, caCertificate, trustAll) } -func CreateClient(timeout time.Duration) HTTPClient { - return &defaultHTTPClient{ +// NewClients creates a new HTTP client the given default timeout. +func NewClient(timeout time.Duration) Client { + return &defaultClient{ client: &http.Client{ Timeout: timeout, Transport: http.DefaultTransport, diff --git a/client/go/internal/util/io.go b/client/go/internal/ioutil/ioutil.go index 9e755737035..d3a33698d13 100644 --- a/client/go/internal/util/io.go +++ b/client/go/internal/ioutil/ioutil.go @@ -2,7 +2,7 @@ // File utilities. // Author: bratseth -package util +package ioutil import ( "bytes" @@ -14,26 +14,26 @@ import ( "strings" ) -// Returns true if the given path exists -func PathExists(path string) bool { +// Exists returns true if the given path exists. +func Exists(path string) bool { info, err := os.Stat(path) return !errors.Is(err, os.ErrNotExist) && info != nil } -// Returns true if the given path points to an existing directory -func IsDirectory(path string) bool { +// IsDir returns true if the given path points to an existing directory. +func IsDir(path string) bool { info, err := os.Stat(path) return !errors.Is(err, os.ErrNotExist) && info != nil && info.IsDir() } -// Returns true if the given path points to an existing file -func IsRegularFile(path string) bool { +// IsFile returns true if the given path points to an existing regular file. +func IsFile(path string) bool { info, err := os.Stat(path) return !errors.Is(err, os.ErrNotExist) && info != nil && info.Mode().IsRegular() } -// Returns true if the given path points to an executable -func IsExecutableFile(path string) bool { +// IsExecutable returns true if the given path points to an executable file. +func IsExecutable(path string) bool { info, err := os.Stat(path) return !errors.Is(err, os.ErrNotExist) && info != nil && @@ -41,21 +41,21 @@ func IsExecutableFile(path string) bool { ((int(info.Mode()) & 0111) == 0111) } -// Returns the content of a reader as a string +// ReaderToString Returns the content of reader as a string. Read errors are ignored. func ReaderToString(reader io.Reader) string { var buffer strings.Builder io.Copy(&buffer, reader) return buffer.String() } -// Returns the content of a reader as a byte array +// ReaderToBytes returns the content of a reader as a byte array. Read errors are ignored. func ReaderToBytes(reader io.Reader) []byte { var buffer bytes.Buffer buffer.ReadFrom(reader) return buffer.Bytes() } -// Returns the contents of reader as indented JSON +// ReaderToJSON returns the contents of reader as indented JSON. Read errors are ignored. func ReaderToJSON(reader io.Reader) string { bodyBytes, _ := io.ReadAll(reader) var prettyJSON bytes.Buffer diff --git a/client/go/internal/util/io_test.go b/client/go/internal/ioutil/ioutil_test.go index 0b2ad0f081b..907132c9eaa 100644 --- a/client/go/internal/util/io_test.go +++ b/client/go/internal/ioutil/ioutil_test.go @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package util +package ioutil import ( "os" @@ -9,46 +9,46 @@ import ( ) func TestPathExists(t *testing.T) { - assert.Equal(t, true, PathExists("io.go")) - assert.Equal(t, false, PathExists("nosuchthing.go")) + assert.Equal(t, true, Exists("ioutil.go")) + assert.Equal(t, false, Exists("nosuchthing.go")) tmpDir := t.TempDir() err := os.MkdirAll(tmpDir+"/no", 0755) assert.Nil(t, err) err = os.MkdirAll(tmpDir+"/no/such", 0) assert.Nil(t, err) - assert.Equal(t, false, PathExists(tmpDir+"/no/such/thing.go")) + assert.Equal(t, false, Exists(tmpDir+"/no/such/thing.go")) } func TestIsDir(t *testing.T) { tmpDir := t.TempDir() err := os.MkdirAll(tmpDir+"/no", 0755) assert.Nil(t, err) - assert.Equal(t, true, IsDirectory(tmpDir+"/no")) + assert.Equal(t, true, IsDir(tmpDir+"/no")) err = os.MkdirAll(tmpDir+"/no/such", 0) assert.Nil(t, err) - assert.Equal(t, true, IsDirectory(tmpDir+"/no/such")) - assert.Equal(t, false, IsDirectory(tmpDir+"/no/such/thing.go")) + assert.Equal(t, true, IsDir(tmpDir+"/no/such")) + assert.Equal(t, false, IsDir(tmpDir+"/no/such/thing.go")) } func TestIsRegularFile(t *testing.T) { - assert.Equal(t, true, IsRegularFile("io.go")) - assert.Equal(t, false, IsRegularFile(".")) + assert.Equal(t, true, IsFile("ioutil.go")) + assert.Equal(t, false, IsFile(".")) tmpDir := t.TempDir() err := os.MkdirAll(tmpDir+"/no", 0755) assert.Nil(t, err) err = os.MkdirAll(tmpDir+"/no/such", 0) assert.Nil(t, err) - assert.Equal(t, false, IsRegularFile(tmpDir+"/no/such/thing.go")) + assert.Equal(t, false, IsFile(tmpDir+"/no/such/thing.go")) } func TestIsExecutableFile(t *testing.T) { - assert.Equal(t, false, IsExecutableFile("io.go")) - assert.Equal(t, false, IsExecutableFile("nosuchthing.go")) + assert.Equal(t, false, IsExecutable("io.go")) + assert.Equal(t, false, IsExecutable("nosuchthing.go")) tmpDir := t.TempDir() err := os.WriteFile(tmpDir+"/run.sh", []byte("#!/bin/sh\necho foo\n"), 0755) assert.Nil(t, err) - assert.Equal(t, true, IsExecutableFile(tmpDir+"/run.sh")) + assert.Equal(t, true, IsExecutable(tmpDir+"/run.sh")) /* unix only: out, err := BackTicksWithStderr.Run(tmpDir + "/run.sh") assert.Nil(t, err) diff --git a/client/go/internal/util/array_list.go b/client/go/internal/list/array_list.go index 0e768b5617f..07427074089 100644 --- a/client/go/internal/util/array_list.go +++ b/client/go/internal/list/array_list.go @@ -1,8 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Author: arnej -// generic utilities -package util +package list type ArrayList[E comparable] []E diff --git a/client/go/internal/util/array_list_test.go b/client/go/internal/list/array_list_test.go index d8a3fa88b5c..f31a94966c5 100644 --- a/client/go/internal/util/array_list_test.go +++ b/client/go/internal/list/array_list_test.go @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package util + +package list import ( "testing" diff --git a/client/go/internal/mock/http.go b/client/go/internal/mock/http.go index c01811c4630..06e143ab80d 100644 --- a/client/go/internal/mock/http.go +++ b/client/go/internal/mock/http.go @@ -9,7 +9,7 @@ import ( "strconv" "time" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" ) type HTTPClient struct { @@ -95,4 +95,4 @@ func (c *HTTPClient) Do(request *http.Request, timeout time.Duration) (*http.Res nil } -func (c *HTTPClient) Clone() util.HTTPClient { return c } +func (c *HTTPClient) Clone() httputil.Client { return c } diff --git a/client/go/internal/util/execvp.go b/client/go/internal/osutil/execvp.go index 38514696365..331b8166428 100644 --- a/client/go/internal/util/execvp.go +++ b/client/go/internal/osutil/execvp.go @@ -3,7 +3,7 @@ //go:build !windows -package util +package osutil import ( "fmt" @@ -12,6 +12,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "golang.org/x/sys/unix" ) @@ -22,7 +23,7 @@ func findInPath(prog string) string { path := strings.Split(os.Getenv(envvars.PATH), ":") for _, dir := range path { fn := dir + "/" + prog - if IsExecutableFile(fn) { + if ioutil.IsExecutable(fn) { return fn } } diff --git a/client/go/internal/util/execvp_windows.go b/client/go/internal/osutil/execvp_windows.go index d01eda589ff..0e8e7a4a673 100644 --- a/client/go/internal/util/execvp_windows.go +++ b/client/go/internal/osutil/execvp_windows.go @@ -3,7 +3,7 @@ //go:build windows -package util +package osutil import ( "fmt" diff --git a/client/go/internal/util/fix_fs.go b/client/go/internal/osutil/fix_fs.go index 12d49462e07..837624cc05b 100644 --- a/client/go/internal/util/fix_fs.go +++ b/client/go/internal/osutil/fix_fs.go @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Author: arnej -package util +package osutil import ( "errors" @@ -149,5 +149,5 @@ func (spec *FixSpec) complainAndExit(got error, fn string, wanted os.FileMode) { out, _ := BackTicksWithStderr.Run("stat", "--", fn) trace.Warning(out) trace.Warning("this is a fatal error!") - JustExitWith(got) + ExitErr(got) } diff --git a/client/go/internal/util/fix_fs_test.go b/client/go/internal/osutil/fix_fs_test.go index 0ecf2e06535..792986d7996 100644 --- a/client/go/internal/util/fix_fs_test.go +++ b/client/go/internal/osutil/fix_fs_test.go @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package util +package osutil import ( "os" @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" ) func setup(t *testing.T) string { @@ -36,15 +37,15 @@ func testFixSpec(t *testing.T, spec FixSpec) { spec.FixFile(tmpDir + "/a/f3") spec.FixFile(tmpDir + "/b/f4") spec.FixFile(tmpDir + "/a/bad/f5") - assert.Equal(t, true, IsDirectory(tmpDir+"/a")) - assert.Equal(t, true, IsDirectory(tmpDir+"/b")) - assert.Equal(t, true, IsDirectory(tmpDir+"/a/bad")) - assert.Equal(t, true, IsDirectory(tmpDir+"/a/bad/ok")) - assert.Equal(t, true, IsRegularFile(tmpDir+"/a/f1")) - assert.Equal(t, true, IsRegularFile(tmpDir+"/a/f2")) - assert.Equal(t, false, IsRegularFile(tmpDir+"/a/f3")) - assert.Equal(t, false, IsRegularFile(tmpDir+"/b/f4")) - assert.Equal(t, false, IsRegularFile(tmpDir+"/a/bad/f5")) + assert.Equal(t, true, ioutil.IsDir(tmpDir+"/a")) + assert.Equal(t, true, ioutil.IsDir(tmpDir+"/b")) + assert.Equal(t, true, ioutil.IsDir(tmpDir+"/a/bad")) + assert.Equal(t, true, ioutil.IsDir(tmpDir+"/a/bad/ok")) + assert.Equal(t, true, ioutil.IsFile(tmpDir+"/a/f1")) + assert.Equal(t, true, ioutil.IsFile(tmpDir+"/a/f2")) + assert.Equal(t, false, ioutil.IsFile(tmpDir+"/a/f3")) + assert.Equal(t, false, ioutil.IsFile(tmpDir+"/b/f4")) + assert.Equal(t, false, ioutil.IsFile(tmpDir+"/a/bad/f5")) info, err := os.Stat(tmpDir + "/a") assert.Nil(t, err) @@ -118,7 +119,7 @@ func TestSuperUserOnly(t *testing.T) { func expectSimplePanic() { if r := recover(); r != nil { - if jee, ok := r.(*JustExitError); ok { + if jee, ok := r.(*ExitError); ok { trace.Trace("got as expected:", jee) return } diff --git a/client/go/internal/util/just_exit.go b/client/go/internal/osutil/just_exit.go index ad07f451c9c..5ad85ec9ceb 100644 --- a/client/go/internal/util/just_exit.go +++ b/client/go/internal/osutil/just_exit.go @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Author: arnej -package util +package osutil import ( "fmt" @@ -9,12 +9,12 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/trace" ) -type JustExitError struct { +type ExitError struct { err error msg string } -func (j *JustExitError) String() string { +func (j *ExitError) String() string { if j.err != nil { if j.msg == "" { return j.err.Error() @@ -27,22 +27,22 @@ func (j *JustExitError) String() string { return j.msg } -func (j *JustExitError) Error() string { +func (j *ExitError) Error() string { return j.String() } -func JustExitMsg(message string) { +func ExitMsg(message string) { trace.Trace("just exit with message") - j := JustExitError{ + j := ExitError{ err: nil, msg: message, } panic(&j) } -func JustExitWith(e error) { +func ExitErr(e error) { trace.Trace("just exit with error") - j := JustExitError{ + j := ExitError{ err: e, msg: "", } diff --git a/client/go/internal/util/run_cmd.go b/client/go/internal/osutil/run_cmd.go index cc40f86154c..ca0d621f9f9 100644 --- a/client/go/internal/util/run_cmd.go +++ b/client/go/internal/osutil/run_cmd.go @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Author: arnej -package util +package osutil import ( "bytes" diff --git a/client/go/internal/util/setrlimit.go b/client/go/internal/osutil/setrlimit.go index 1a96d260fcb..6bc6d68af3e 100644 --- a/client/go/internal/util/setrlimit.go +++ b/client/go/internal/osutil/setrlimit.go @@ -2,7 +2,7 @@ //go:build !windows -package util +package osutil import ( "os" diff --git a/client/go/internal/util/setrlimit_windows.go b/client/go/internal/osutil/setrlimit_windows.go index f2993c7af13..e61233ba9e6 100644 --- a/client/go/internal/util/setrlimit_windows.go +++ b/client/go/internal/osutil/setrlimit_windows.go @@ -2,7 +2,7 @@ //go:build windows -package util +package osutil type ResourceId int diff --git a/client/go/internal/util/tune_logctl.go b/client/go/internal/osutil/tune_logctl.go index b66c14c2d65..f68259170c7 100644 --- a/client/go/internal/util/tune_logctl.go +++ b/client/go/internal/osutil/tune_logctl.go @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Author: arnej -package util +package osutil func TuneLogging(serviceName, component, settings string) bool { arg := serviceName diff --git a/client/go/internal/util/tuning.go b/client/go/internal/osutil/tuning.go index cca314247ab..8e9b894e8ae 100644 --- a/client/go/internal/util/tuning.go +++ b/client/go/internal/osutil/tuning.go @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Author: arnej -package util +package osutil import ( "os" diff --git a/client/go/internal/util/md5.go b/client/go/internal/util/md5.go deleted file mode 100644 index 6a98b49c472..00000000000 --- a/client/go/internal/util/md5.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package util - -import ( - "crypto/md5" - "fmt" - "io" -) - -func Md5Hex(text string) string { - hasher := md5.New() - io.WriteString(hasher, text) - hash := hasher.Sum(nil) - return fmt.Sprintf("%x", hash) -} diff --git a/client/go/internal/util/spinner.go b/client/go/internal/util/spinner.go deleted file mode 100644 index 323a5fffe12..00000000000 --- a/client/go/internal/util/spinner.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package util - -import ( - "io" - "strings" - "time" - - "github.com/briandowns/spinner" -) - -// Spinner writes message to writer w and executes function fn. While fn is running a spinning animation will be -// displayed after message. -func Spinner(w io.Writer, message string, fn func() error) error { - s := spinner.New(spinner.CharSets[11], 100*time.Millisecond, spinner.WithWriter(w)) - // Cursor is hidden by default. Hiding cursor requires Stop() to be called to restore cursor (i.e. if the process is - // interrupted), however we don't want to bother with a signal handler just for this - s.HideCursor = false - if err := s.Color("blue", "bold"); err != nil { - return err - } - if !strings.HasSuffix(message, " ") { - message += " " - } - s.Prefix = message - s.FinalMSG = "\r" + message + "done\n" - s.Start() - err := fn() - if err != nil { - s.FinalMSG = "\r" + message + "failed\n" - } - s.Stop() - return err -} diff --git a/client/go/internal/version/version.go b/client/go/internal/version/version.go index 1b27d01ea83..513c808a582 100644 --- a/client/go/internal/version/version.go +++ b/client/go/internal/version/version.go @@ -5,8 +5,6 @@ import ( "fmt" "strconv" "strings" - - "github.com/vespa-engine/vespa/client/go/internal/util" ) // Version represents a semantic version number. @@ -74,7 +72,7 @@ func (v1 Version) Less(v2 Version) bool { return v1.Compare(v2) < 0 } func MustParse(s string) Version { v, err := Parse(s) if err != nil { - util.JustExitWith(err) + panic(err) } return v } diff --git a/client/go/internal/vespa/application.go b/client/go/internal/vespa/application.go index 6d28b24100f..5d1ab610e38 100644 --- a/client/go/internal/vespa/application.go +++ b/client/go/internal/vespa/application.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" ) type ApplicationPackage struct { @@ -24,7 +24,7 @@ func (ap *ApplicationPackage) HasDeploymentSpec() bool { return ap.hasFile("depl func (ap *ApplicationPackage) hasFile(pathSegment ...string) bool { if !ap.IsZip() { - return util.PathExists(filepath.Join(append([]string{ap.Path}, pathSegment...)...)) + return ioutil.Exists(filepath.Join(append([]string{ap.Path}, pathSegment...)...)) } zipName := filepath.Join(pathSegment...) return ap.hasZipEntry(func(name string) bool { return zipName == name }) @@ -50,7 +50,7 @@ func (ap *ApplicationPackage) IsJava() bool { if ap.IsZip() { return ap.hasZipEntry(func(name string) bool { return filepath.Ext(name) == ".jar" }) } - return util.PathExists(filepath.Join(ap.Path, "pom.xml")) + return ioutil.Exists(filepath.Join(ap.Path, "pom.xml")) } func (ap *ApplicationPackage) Validate() error { @@ -74,11 +74,11 @@ func (ap *ApplicationPackage) Validate() error { func isZip(filename string) bool { return filepath.Ext(filename) == ".zip" } func zipDir(dir string, destination string) error { - if !util.PathExists(dir) { + if !ioutil.Exists(dir) { message := "'" + dir + "' should be an application package zip or dir, but does not exist" return errors.New(message) } - if !util.IsDirectory(dir) { + if !ioutil.IsDir(dir) { message := "'" + dir + "' should be an application package dir, but is a (non-zip) file" return errors.New(message) } @@ -267,10 +267,10 @@ func findApplicationPackage(zipOrDir string, options PackageOptions) (Applicatio } // Pre-packaged application. We prefer the uncompressed application because this allows us to add // security/clients.pem to the package on-demand - hasPOM := util.PathExists(filepath.Join(zipOrDir, "pom.xml")) + hasPOM := ioutil.Exists(filepath.Join(zipOrDir, "pom.xml")) if hasPOM && !options.SourceOnly { path := filepath.Join(zipOrDir, "target", "application") - if util.PathExists(path) { + if ioutil.Exists(path) { testPath := existingPath(filepath.Join(zipOrDir, "target", "application-test")) return ApplicationPackage{Path: path, TestPath: testPath}, nil } @@ -279,14 +279,14 @@ func findApplicationPackage(zipOrDir string, options PackageOptions) (Applicatio } } // Application with Maven directory structure, but with no POM or no hard requirement on packaging - if path := filepath.Join(zipOrDir, "src", "main", "application"); util.PathExists(path) { + if path := filepath.Join(zipOrDir, "src", "main", "application"); ioutil.Exists(path) { testPath := existingPath(filepath.Join(zipOrDir, "src", "test", "application")) return ApplicationPackage{Path: path, TestPath: testPath}, nil } // Application without Java components - if util.PathExists(filepath.Join(zipOrDir, "services.xml")) { + if ioutil.Exists(filepath.Join(zipOrDir, "services.xml")) { testPath := "" - if util.PathExists(filepath.Join(zipOrDir, "tests")) { + if ioutil.Exists(filepath.Join(zipOrDir, "tests")) { testPath = zipOrDir } return ApplicationPackage{Path: zipOrDir, TestPath: testPath}, nil @@ -295,7 +295,7 @@ func findApplicationPackage(zipOrDir string, options PackageOptions) (Applicatio } func existingPath(path string) string { - if util.PathExists(path) { + if ioutil.Exists(path) { return path } return "" diff --git a/client/go/internal/vespa/crypto.go b/client/go/internal/vespa/crypto.go index 13d3ac570cc..9b4d776d97d 100644 --- a/client/go/internal/vespa/crypto.go +++ b/client/go/internal/vespa/crypto.go @@ -20,7 +20,7 @@ import ( "strings" "time" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" ) const ( @@ -36,18 +36,18 @@ type PemKeyPair struct { // WriteCertificateFile writes the certificate contained in this key pair to certificateFile. func (kp *PemKeyPair) WriteCertificateFile(certificateFile string, overwrite bool) error { - if util.PathExists(certificateFile) && !overwrite { + if ioutil.Exists(certificateFile) && !overwrite { return fmt.Errorf("cannot overwrite existing file: %s", certificateFile) } - return util.AtomicWriteFile(certificateFile, kp.Certificate) + return ioutil.AtomicWriteFile(certificateFile, kp.Certificate) } // WritePrivateKeyFile writes the private key contained in this key pair to privateKeyFile. func (kp *PemKeyPair) WritePrivateKeyFile(privateKeyFile string, overwrite bool) error { - if util.PathExists(privateKeyFile) && !overwrite { + if ioutil.Exists(privateKeyFile) && !overwrite { return fmt.Errorf("cannot overwrite existing file: %s", privateKeyFile) } - return util.AtomicWriteFile(privateKeyFile, kp.PrivateKey) + return ioutil.AtomicWriteFile(privateKeyFile, kp.PrivateKey) } // CreateKeyPair creates a key pair containing a private key and self-signed X509 certificate. diff --git a/client/go/internal/vespa/deploy.go b/client/go/internal/vespa/deploy.go index 35fd523f15a..10ddb321e19 100644 --- a/client/go/internal/vespa/deploy.go +++ b/client/go/internal/vespa/deploy.go @@ -18,7 +18,7 @@ import ( "strings" "time" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/version" ) @@ -120,10 +120,10 @@ func ZoneFromString(s string) (ZoneID, error) { } func Fetch(deployment DeploymentOptions, path string) (string, error) { - if util.IsDirectory(path) { + if ioutil.IsDir(path) { path = filepath.Join(path, "application.zip") } - if util.PathExists(path) { + if ioutil.Exists(path) { return "", fmt.Errorf("%s already exists", path) } if deployment.Target.IsCloud() { @@ -540,7 +540,7 @@ func checkResponse(req *http.Request, response *http.Response) error { if response.StatusCode/100 == 4 { return fmt.Errorf("invalid application package (%s)\n%s", response.Status, extractError(response.Body)) } else if response.StatusCode != 200 { - return fmt.Errorf("error from deploy API at %s (%s):\n%s", req.URL.Host, response.Status, util.ReaderToJSON(response.Body)) + return fmt.Errorf("error from deploy API at %s (%s):\n%s", req.URL.Host, response.Status, ioutil.ReaderToJSON(response.Body)) } return nil } diff --git a/client/go/internal/vespa/deploy_test.go b/client/go/internal/vespa/deploy_test.go index 516a21e7786..4c2fb912224 100644 --- a/client/go/internal/vespa/deploy_test.go +++ b/client/go/internal/vespa/deploy_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" "github.com/vespa-engine/vespa/client/go/internal/mock" - "github.com/vespa-engine/vespa/client/go/internal/util" "github.com/vespa-engine/vespa/client/go/internal/version" ) @@ -235,7 +235,7 @@ func TestFetch(t *testing.T) { dir := t.TempDir() dst, err := Fetch(opts, dir) require.Nil(t, err) - assert.True(t, util.PathExists(dst)) + assert.True(t, ioutil.Exists(dst)) f, err := os.Open(dst) require.Nil(t, err) @@ -264,7 +264,7 @@ func TestFetchCloud(t *testing.T) { dir := t.TempDir() dst, err := Fetch(opts, dir) require.Nil(t, err) - assert.True(t, util.PathExists(dst)) + assert.True(t, ioutil.Exists(dst)) } type pkgFixture struct { diff --git a/client/go/internal/vespa/detect_hostname.go b/client/go/internal/vespa/detect_hostname.go index e6b2d113ec1..062d83d66a4 100644 --- a/client/go/internal/vespa/detect_hostname.go +++ b/client/go/internal/vespa/detect_hostname.go @@ -11,7 +11,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) // detect if this host is IPv6-only, in which case we want to pass @@ -109,7 +109,7 @@ func findOurHostnameFrom(name string) (string, error) { if good { return trimmed, nil } - backticks := util.BackTicksIgnoreStderr + backticks := osutil.BackTicksIgnoreStderr out, err := backticks.Run("vespa-detect-hostname") if err != nil { out, err = backticks.Run("hostname", "-f") diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index f878938d6fc..3871ab19edd 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -18,7 +18,7 @@ import ( "github.com/go-json-experiment/json" "github.com/klauspost/compress/gzip" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" ) type Compression int @@ -52,7 +52,7 @@ type ClientOptions struct { } type countingHTTPClient struct { - client util.HTTPClient + client httputil.Client inflight atomic.Int64 } @@ -70,7 +70,7 @@ type pendingDocument struct { err error } -func NewClient(options ClientOptions, httpClients []util.HTTPClient) (*Client, error) { +func NewClient(options ClientOptions, httpClients []httputil.Client) (*Client, error) { if len(httpClients) < 1 { return nil, fmt.Errorf("need at least one HTTP client") } diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index b2c1139f95f..89e9e96064b 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -10,8 +10,8 @@ import ( "testing" "time" + "github.com/vespa-engine/vespa/client/go/internal/httputil" "github.com/vespa-engine/vespa/client/go/internal/mock" - "github.com/vespa-engine/vespa/client/go/internal/util" ) type manualClock struct { @@ -33,7 +33,7 @@ type mockHTTPClient struct { func TestLeastBusyClient(t *testing.T) { httpClient := mock.HTTPClient{} - var httpClients []util.HTTPClient + var httpClients []httputil.Client for i := 0; i < 4; i++ { httpClients = append(httpClients, &mockHTTPClient{i, &httpClient}) } @@ -83,7 +83,7 @@ func TestClientSend(t *testing.T) { client, _ := NewClient(ClientOptions{ BaseURL: "https://example.com:1337", Timeout: time.Duration(5 * time.Second), - }, []util.HTTPClient{&httpClient}) + }, []httputil.Client{&httpClient}) clock := manualClock{t: time.Now(), tick: time.Second} client.now = clock.now var stats Stats @@ -164,7 +164,7 @@ func TestClientGet(t *testing.T) { client, _ := NewClient(ClientOptions{ BaseURL: "https://example.com:1337", Timeout: time.Duration(5 * time.Second), - }, []util.HTTPClient{&httpClient}) + }, []httputil.Client{&httpClient}) clock := manualClock{t: time.Now(), tick: time.Second} client.now = clock.now doc := `{ @@ -196,7 +196,7 @@ func TestClientSendCompressed(t *testing.T) { client, _ := NewClient(ClientOptions{ BaseURL: "https://example.com:1337", Timeout: time.Duration(5 * time.Second), - }, []util.HTTPClient{httpClient}) + }, []httputil.Client{httpClient}) bigBody := fmt.Sprintf(`{"fields": {"foo": "%s"}}`, strings.Repeat("s", 512+1)) bigDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(bigBody)} @@ -313,7 +313,7 @@ func TestClientMethodAndURL(t *testing.T) { httpClient := mock.HTTPClient{} client, _ := NewClient(ClientOptions{ BaseURL: "https://example.com/", - }, []util.HTTPClient{&httpClient}) + }, []httputil.Client{&httpClient}) for i, tt := range tests { client.options.Timeout = tt.options.Timeout client.options.Route = tt.options.Route @@ -333,7 +333,7 @@ func benchmarkClientSend(b *testing.B, compression Compression, document Documen Compression: compression, BaseURL: "https://example.com:1337", Timeout: time.Duration(5 * time.Second), - }, []util.HTTPClient{&httpClient}) + }, []httputil.Client{&httpClient}) b.ResetTimer() // ignore setup for n := 0; n < b.N; n++ { client.Send(document) diff --git a/client/go/internal/vespa/find_home.go b/client/go/internal/vespa/find_home.go index 46dcdedd71c..3545ce6a8c7 100644 --- a/client/go/internal/vespa/find_home.go +++ b/client/go/internal/vespa/find_home.go @@ -11,7 +11,8 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) const ( @@ -46,7 +47,7 @@ func FindHome() string { } for _, dir := range strings.Split(os.Getenv(envvars.PATH), ":") { fn := fmt.Sprintf("%s/%s", dir, myProgName) - if util.IsRegularFile(fn) { + if ioutil.IsFile(fn) { trace.Debug("findPath", myProgName, "=>", dir) return dir } @@ -56,7 +57,7 @@ func FindHome() string { // detect path from argv[0] for path := findPath(); path != ""; path = dirName(path) { mySelf := fmt.Sprintf("%s/%s", path, scriptUtilsFilename) - if util.IsRegularFile(mySelf) { + if ioutil.IsFile(mySelf) { trace.Debug("found", mySelf, "VH =>", path) os.Setenv(envvars.VESPA_HOME, path) return path @@ -82,9 +83,9 @@ func HasFileUnderVespaHome(fn string) (bool, string) { func FindAndVerifyVespaHome() string { vespaHome := FindHome() myself := fmt.Sprintf("%s/%s", vespaHome, scriptUtilsFilename) - if !util.IsExecutableFile(myself) { + if !ioutil.IsExecutable(myself) { trace.Warning("missing or bad file:", myself) - util.JustExitMsg("Not a valid VESPA_HOME: " + vespaHome) + osutil.ExitMsg("Not a valid VESPA_HOME: " + vespaHome) } return vespaHome } diff --git a/client/go/internal/vespa/load_env.go b/client/go/internal/vespa/load_env.go index 24e1b1cdefa..a799cbf4f9a 100644 --- a/client/go/internal/vespa/load_env.go +++ b/client/go/internal/vespa/load_env.go @@ -14,7 +14,8 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/ioutil" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) const ( @@ -248,7 +249,7 @@ func shellQuote(s string) string { } if i != l { err := fmt.Errorf("expected length %d but was %d", l, i) - util.JustExitWith(err) + osutil.ExitErr(err) } return string(res) } @@ -276,7 +277,7 @@ func (builder *pathBuilder) applyTo(receiver loadEnvReceiver) { } func (builder *pathBuilder) appendPath(p string) { - if !util.IsDirectory(p) { + if !ioutil.IsDir(p) { return } for _, elem := range builder.curPath { diff --git a/client/go/internal/vespa/prestart.go b/client/go/internal/vespa/prestart.go index 5b29915bcb2..e6932e951ad 100644 --- a/client/go/internal/vespa/prestart.go +++ b/client/go/internal/vespa/prestart.go @@ -8,7 +8,7 @@ import ( "os" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) func RunPreStart() error { @@ -23,7 +23,7 @@ func RunPreStart() error { return err } vespaUid, vespaGid := FindVespaUidAndGid() - fixSpec := util.FixSpec{ + fixSpec := osutil.FixSpec{ UserId: vespaUid, GroupId: vespaGid, DirMode: 0755, @@ -57,7 +57,7 @@ func RunPreStart() error { // fix wrong ownerships within directories: var fixer fs.WalkDirFunc = func(path string, d fs.DirEntry, err error) error { if err != nil { - util.JustExitWith(err) + osutil.ExitErr(err) } if d.IsDir() { fixSpec.FixDir(path) diff --git a/client/go/internal/vespa/switch_user.go b/client/go/internal/vespa/switch_user.go index c84da567224..bce59b96f32 100644 --- a/client/go/internal/vespa/switch_user.go +++ b/client/go/internal/vespa/switch_user.go @@ -11,7 +11,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/osutil" ) const ENV_CHECK = envvars.VESPA_ALREADY_SWITCHED_USER_TO @@ -55,7 +55,7 @@ func CheckCorrectUser() { if err2 != nil { trace.Warning("note: user.Lookup(", vespaUser, ") failed:", err2) } - util.JustExitMsg("running as wrong user. Check your VESPA_USER setting") + osutil.ExitMsg("running as wrong user. Check your VESPA_USER setting") } // re-execute a vespa-wrapper action after switching to the vespa user @@ -87,7 +87,7 @@ func MaybeSwitchUser(action string) error { mySelf := fmt.Sprintf("%s/%s", vespaHome, scriptUtilsFilename) os.Setenv(ENV_CHECK, wantUser.Username) args := []string{SU_PROG, mySelf, action} - return util.Execvp(SU_PROG, args) + return osutil.Execvp(SU_PROG, args) } return nil } diff --git a/client/go/internal/vespa/target.go b/client/go/internal/vespa/target.go index 3a76eac0292..543ce2f4a29 100644 --- a/client/go/internal/vespa/target.go +++ b/client/go/internal/vespa/target.go @@ -12,7 +12,7 @@ import ( "time" "github.com/vespa-engine/vespa/client/go/internal/curl" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" "github.com/vespa-engine/vespa/client/go/internal/version" ) @@ -84,7 +84,7 @@ type Service struct { deployAPI bool auth Authenticator - httpClient util.HTTPClient + httpClient httputil.Client customClient bool retryInterval time.Duration } @@ -143,7 +143,7 @@ type LogOptions struct { func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { if !s.customClient { // Do not override TLS config if a custom client has been configured - util.ConfigureTLS(s.httpClient, s.TLSOptions.KeyPair, s.TLSOptions.CACertificate, s.TLSOptions.TrustAll) + httputil.ConfigureTLS(s.httpClient, s.TLSOptions.KeyPair, s.TLSOptions.CACertificate, s.TLSOptions.TrustAll) } if s.auth != nil { if err := s.auth.Authenticate(request); err != nil { @@ -157,7 +157,7 @@ func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Respon } // SetClient sets a custom HTTP client that this service should use. -func (s *Service) SetClient(client util.HTTPClient) { +func (s *Service) SetClient(client httputil.Client) { s.httpClient = client s.customClient = true } diff --git a/client/go/internal/vespa/target_cloud.go b/client/go/internal/vespa/target_cloud.go index 708810061d2..c063b99edef 100644 --- a/client/go/internal/vespa/target_cloud.go +++ b/client/go/internal/vespa/target_cloud.go @@ -12,7 +12,7 @@ import ( "strconv" "time" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" "github.com/vespa-engine/vespa/client/go/internal/version" ) @@ -35,7 +35,7 @@ type cloudTarget struct { apiOptions APIOptions deploymentOptions CloudDeploymentOptions logOptions LogOptions - httpClient util.HTTPClient + httpClient httputil.Client apiAuth Authenticator deploymentAuth Authenticator retryInterval time.Duration @@ -74,7 +74,7 @@ type logMessage struct { } // CloudTarget creates a Target for the Vespa Cloud or hosted Vespa platform. -func CloudTarget(httpClient util.HTTPClient, apiAuth Authenticator, deploymentAuth Authenticator, +func CloudTarget(httpClient httputil.Client, apiAuth Authenticator, deploymentAuth Authenticator, apiOptions APIOptions, deploymentOptions CloudDeploymentOptions, logOptions LogOptions, retryInterval time.Duration) (Target, error) { return &cloudTarget{ diff --git a/client/go/internal/vespa/target_custom.go b/client/go/internal/vespa/target_custom.go index 0f3817e8c13..9d62f7dc297 100644 --- a/client/go/internal/vespa/target_custom.go +++ b/client/go/internal/vespa/target_custom.go @@ -11,14 +11,14 @@ import ( "strconv" "time" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/httputil" "github.com/vespa-engine/vespa/client/go/internal/version" ) type customTarget struct { targetType string baseURL string - httpClient util.HTTPClient + httpClient httputil.Client tlsOptions TLSOptions retryInterval time.Duration } @@ -36,7 +36,7 @@ type serviceInfo struct { } // LocalTarget creates a target for a Vespa platform running locally. -func LocalTarget(httpClient util.HTTPClient, tlsOptions TLSOptions, retryInterval time.Duration) Target { +func LocalTarget(httpClient httputil.Client, tlsOptions TLSOptions, retryInterval time.Duration) Target { return &customTarget{ targetType: TargetLocal, baseURL: "http://127.0.0.1", @@ -47,7 +47,7 @@ func LocalTarget(httpClient util.HTTPClient, tlsOptions TLSOptions, retryInterva } // CustomTarget creates a Target for a Vespa platform running at baseURL. -func CustomTarget(httpClient util.HTTPClient, baseURL string, tlsOptions TLSOptions, retryInterval time.Duration) Target { +func CustomTarget(httpClient httputil.Client, baseURL string, tlsOptions TLSOptions, retryInterval time.Duration) Target { return &customTarget{ targetType: TargetCustom, baseURL: baseURL, diff --git a/client/go/internal/vespa/xml/config.go b/client/go/internal/vespa/xml/config.go index 05d73474ffc..d1c16b01652 100644 --- a/client/go/internal/vespa/xml/config.go +++ b/client/go/internal/vespa/xml/config.go @@ -11,7 +11,6 @@ import ( "strconv" "strings" - "github.com/vespa-engine/vespa/client/go/internal/util" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -25,7 +24,7 @@ func init() { </deployment>` d, err := ReadDeployment(strings.NewReader(defaultDeploymentRaw)) if err != nil { - util.JustExitWith(err) + panic(err) } DefaultDeployment = d } diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index bc7c90f361e..984bdbd2f48 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -5645,9 +5645,9 @@ v8-to-istanbul@^9.0.1: convert-source-map "^1.6.0" vite@^5.0.0: - version "5.0.3" - resolved "https://registry.yarnpkg.com/vite/-/vite-5.0.3.tgz#febf6801604c618234de331bd04382cf9a149ec6" - integrity sha512-WgEq8WEKpZ8c0DL4M1+E+kBZEJyjBmGVrul6z8Ljfhv+PPbNF4aGq014DwNYxGz2FGq6NKL0N8usdiESWd2l2w== + version "5.0.4" + resolved "https://registry.yarnpkg.com/vite/-/vite-5.0.4.tgz#d984d2aaa8bac30f1ca9b9eea9b97e052f88c307" + integrity sha512-RzAr8LSvM8lmhB4tQ5OPcBhpjOZRZjuxv9zO5UcxeoY2bd3kP3Ticd40Qma9/BqZ8JS96Ll/jeBX9u+LJZrhVg== dependencies: esbuild "^0.19.3" postcss "^8.4.31" diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml index 8bdfb25e221..7f845a26c73 100644 --- a/clustercontroller-core/pom.xml +++ b/clustercontroller-core/pom.xml @@ -137,6 +137,10 @@ <configuration> <forkCount>4</forkCount> <rerunFailingTestsCount>5</rerunFailingTestsCount> + <systemPropertyVariables> + <!-- Avoid 64 MiB default in favor of just 64 KiB --> + <zookeeper.preAllocSize>64</zookeeper.preAllocSize> + </systemPropertyVariables> </configuration> </plugin> </plugins> diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/DefinedFlag.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/DefinedFlag.java index 84ba451ac7f..c6e51e530b1 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/DefinedFlag.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/DefinedFlag.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.vespa.flags.FlagDefinition; -import com.yahoo.vespa.flags.json.DimensionHelper; import java.io.IOException; import java.io.OutputStream; @@ -42,7 +41,7 @@ public class DefinedFlag extends HttpResponse { definitionNode.put("createdAt", flagDefinition.getCreatedAt().toString()); definitionNode.put("expiresAt", flagDefinition.getExpiresAt().toString()); ArrayNode dimensionsNode = definitionNode.putArray("dimensions"); - flagDefinition.getDimensions().forEach(dimension -> dimensionsNode.add(DimensionHelper.toWire(dimension))); + flagDefinition.getDimensions().forEach(dimension -> dimensionsNode.add(dimension.toWire())); } @Override diff --git a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImplTest.java b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImplTest.java index a82a6f7c599..c9c11e2b0e2 100644 --- a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImplTest.java +++ b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImplTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.configserver.flags.db; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.JsonNodeRawFlag; @@ -28,11 +29,11 @@ public class FlagsDbImplTest { MockCurator curator = new MockCurator(); FlagsDbImpl db = new FlagsDbImpl(curator); - var params = new Condition.CreateParams(FetchVector.Dimension.HOSTNAME).withValues("host1"); + var params = new Condition.CreateParams(Dimension.HOSTNAME).withValues("host1"); Condition condition1 = WhitelistCondition.create(params); Rule rule1 = new Rule(Optional.of(JsonNodeRawFlag.fromJson("13")), condition1); FlagId flagId = new FlagId("id"); - FlagData data = new FlagData(flagId, new FetchVector().with(FetchVector.Dimension.ZONE_ID, "zone-a"), rule1); + FlagData data = new FlagData(flagId, new FetchVector().with(Dimension.ZONE_ID, "zone-a"), rule1); db.setValue(flagId, data); Optional<FlagData> dataCopy = db.getValue(flagId); @@ -43,7 +44,7 @@ public class FlagsDbImplTest { dataCopy.get().serializeToJson()); FlagId flagId2 = new FlagId("id2"); - FlagData data2 = new FlagData(flagId2, new FetchVector().with(FetchVector.Dimension.ZONE_ID, "zone-a"), rule1); + FlagData data2 = new FlagData(flagId2, new FetchVector().with(Dimension.ZONE_ID, "zone-a"), rule1); db.setValue(flagId2, data2); Map<FlagId, FlagData> flags = db.getAllFlagData(); assertEquals(flags.size(), 2); diff --git a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java index 3d37117c002..1e477a6da6e 100644 --- a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java +++ b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java @@ -8,7 +8,7 @@ import com.yahoo.text.Utf8; import com.yahoo.vespa.configserver.flags.FlagsDb; import com.yahoo.vespa.configserver.flags.db.FlagsDbImpl; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.UnboundBooleanFlag; @@ -34,7 +34,7 @@ public class FlagsHandlerTest { "id1", false, List.of("joe"), "2010-01-01", "2030-01-01", "desc1", "mod1"); private static final UnboundBooleanFlag FLAG2 = Flags.defineFeatureFlag( "id2", true, List.of("joe"), "2010-01-01", "2030-01-01", "desc2", "mod2", - FetchVector.Dimension.HOSTNAME, FetchVector.Dimension.INSTANCE_ID); + Dimension.HOSTNAME, Dimension.INSTANCE_ID); private final FlagsDb flagsDb = new FlagsDbImpl(new MockCurator()); private final FlagsHandler handler = new FlagsHandler(FlagsHandler.testContext(), flagsDb); @@ -54,7 +54,7 @@ public class FlagsHandlerTest { void testDefined() { try (Flags.Replacer replacer = Flags.clearFlagsForTesting()) { fixUnusedWarning(replacer); - Flags.defineFeatureFlag("id", false, List.of("joe"), "2010-01-01", "2030-01-01", "desc", "mod", FetchVector.Dimension.HOSTNAME); + Flags.defineFeatureFlag("id", false, List.of("joe"), "2010-01-01", "2030-01-01", "desc", "mod", Dimension.HOSTNAME); verifySuccessfulRequest(Method.GET, "/defined", "", "{\"id\":{\"description\":\"desc\",\"modification-effect\":\"mod\",\"owners\":[\"joe\"],\"createdAt\":\"2010-01-01T00:00:00Z\",\"expiresAt\":\"2030-01-01T00:00:00Z\",\"dimensions\":[\"hostname\"]}}"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 5ac1b685b97..b83a8290cac 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -48,7 +48,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; -import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID; +import static com.yahoo.vespa.flags.Dimension.INSTANCE_ID; import static java.util.stream.Collectors.toSet; /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 76af53eba90..59266dfffaf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -28,7 +28,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.tenant.SecretStoreExternalIdRetriever; -import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; @@ -46,7 +46,7 @@ import java.util.concurrent.ExecutorService; import java.util.function.Predicate; import static com.yahoo.vespa.config.server.ConfigServerSpec.fromConfig; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE; +import static com.yahoo.vespa.flags.Dimension.CLUSTER_TYPE; /** * Implementation of {@link ModelContext} for configserver. @@ -307,10 +307,10 @@ public class ModelContextImpl implements ModelContext { private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) - .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) - .with(FetchVector.Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) - .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) - .with(FetchVector.Dimension.TENANT_ID, appId.tenant().value()) + .with(Dimension.INSTANCE_ID, appId.serializedForm()) + .with(Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) + .with(Dimension.VESPA_VERSION, vespaVersion.toFullString()) + .with(Dimension.TENANT_ID, appId.tenant().value()) .boxedValue(); } @@ -320,10 +320,10 @@ public class ModelContextImpl implements ModelContext { ClusterSpec.Type clusterType, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) - .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) - .with(FetchVector.Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) - .with(FetchVector.Dimension.CLUSTER_TYPE, clusterType.name()) - .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) + .with(Dimension.INSTANCE_ID, appId.serializedForm()) + .with(Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) + .with(Dimension.CLUSTER_TYPE, clusterType.name()) + .with(Dimension.VESPA_VERSION, vespaVersion.toFullString()) .boxedValue(); } @@ -333,10 +333,10 @@ public class ModelContextImpl implements ModelContext { ClusterSpec.Id clusterId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) - .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) - .with(FetchVector.Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) - .with(FetchVector.Dimension.CLUSTER_ID, clusterId.value()) - .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) + .with(Dimension.INSTANCE_ID, appId.serializedForm()) + .with(Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) + .with(Dimension.CLUSTER_ID, clusterId.value()) + .with(Dimension.VESPA_VERSION, vespaVersion.toFullString()) .boxedValue(); } @@ -411,8 +411,8 @@ public class ModelContextImpl implements ModelContext { this.tenantSecretStores = tenantSecretStores; this.secretStore = secretStore; this.jvmGCOptionsFlag = PermanentFlags.JVM_GC_OPTIONS.bindTo(flagSource) - .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()) - .with(FetchVector.Dimension.APPLICATION, applicationId.toSerializedFormWithoutInstance()); + .with(Dimension.INSTANCE_ID, applicationId.serializedForm()) + .with(Dimension.APPLICATION, applicationId.toSerializedFormWithoutInstance()); this.allowDisableMtls = flagValue(flagSource, applicationId, PermanentFlags.ALLOW_DISABLE_MTLS); this.operatorCertificates = operatorCertificates; this.tlsCiphersOverride = flagValue(flagSource, applicationId, PermanentFlags.TLS_CIPHERS_OVERRIDE); @@ -523,8 +523,8 @@ public class ModelContextImpl implements ModelContext { private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) - .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) - .with(FetchVector.Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) + .with(Dimension.INSTANCE_ID, appId.serializedForm()) + .with(Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) .boxedValue(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 99d5d23a87c..47b8215b52d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -83,7 +83,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; -import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID; +import static com.yahoo.vespa.flags.Dimension.INSTANCE_ID; import static java.nio.file.Files.readAttributes; /** diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java index dee4e3dcfbb..d570747ad11 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java @@ -90,7 +90,7 @@ public final class MbusClient extends AbstractResource implements ClientProvider @Override public void handleReply(final Reply reply) { reply.getTrace().trace(6, "Reply received by MbusClient."); - final ResponseHandler handler = (ResponseHandler)reply.getContext(); + ResponseHandler handler = (ResponseHandler) reply.getContext(); reply.popHandler(); // restore user context try { handler.handleResponse(new MbusResponse(StatusCodes.fromMbusReply(reply), reply)) diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java index 4c21489ded2..42fbec7711d 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java @@ -67,6 +67,8 @@ public final class MbusServer extends AbstractResource implements ServerProvider @Override protected void destroy() { log.log(Level.INFO, "Destroying message bus server: " + session.name()); + if (runState.get() == State.RUNNING) + log.log(Level.WARNING, "Message bus server destroyed before being disconnected: " + session.name()); runState.set(State.STOPPED); sessionReference.close(); } @@ -79,6 +81,7 @@ public final class MbusServer extends AbstractResource implements ServerProvider return; } if (state == State.STOPPED) { + log.log(Level.WARNING, "Message bus server received message after being stopped: " + session.name()); dispatchErrorReply(msg, ErrorCode.NETWORK_SHUTDOWN, "MBusServer has been closed."); return; } diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java index 1b3a9afff7b..2283f703463 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java @@ -109,4 +109,5 @@ public class SharedIntermediateSession extends AbstractResource session.destroy(); mbusReference.close(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java index 104db4ea731..90e9897556f 100755 --- a/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java @@ -31,7 +31,9 @@ import java.util.concurrent.ThreadLocalRandom; * <li>rate.id - (String) the id of the client from rate limiting perspective * <li>rate.cost - (Double) the cost Double of this query. This is read after executing the query and hence can be set * by downstream searchers inspecting the result to allow differencing the cost of various queries. Default is 1. - * <li>rate.quota - (Double) the cost per second a particular id is allowed to consume in this system. + * <li>rate.quota - (Double) the cost per second a particular id is allowed to consume. By default this is across + * all nodes of the cluster (i.e, this is invariant with cluster size), set the config variable + * localRate to true to make this be rate per node. * <li>rate.idDimension - (String) the name of the rate-id dimension used when logging metrics. * If this is not specified, the metric will be logged without dimensions. * <li>rate.dryRun - (Boolean) emit metrics on rejected requests but don't actually reject them @@ -69,6 +71,8 @@ public class RateLimitingSearcher extends Searcher { /** Shared capacity across all threads. Each thread will ask for more capacity from here when they run out. */ private final AvailableCapacity availableCapacity; + private final boolean localRate; + /** Capacity already allocated to this thread */ private final ThreadLocal<Map<String, Double>> allocatedCapacity = new ThreadLocal<>(); @@ -90,10 +94,14 @@ public class RateLimitingSearcher extends Searcher { } /** For testing - allows injection of a timer to avoid depending on the system clock */ - public RateLimitingSearcher(RateLimitingConfig rateLimitingConfig, ClusterInfoConfig clusterInfoConfig, MetricReceiver metric, Clock clock) { + public RateLimitingSearcher(RateLimitingConfig rateLimitingConfig, + ClusterInfoConfig clusterInfoConfig, + MetricReceiver metric, + Clock clock) { this.capacityIncrement = rateLimitingConfig.capacityIncrement(); this.recheckForCapacityProbability = rateLimitingConfig.recheckForCapacityProbability(); this.availableCapacity = new AvailableCapacity(rateLimitingConfig.maxAvailableCapacity(), clock); + this.localRate = rateLimitingConfig.localRate(); this.nodeCount = clusterInfoConfig.nodeCount(); @@ -109,7 +117,8 @@ public class RateLimitingSearcher extends Searcher { return execution.search(query); } - rate = rate / nodeCount; + if ( ! localRate) + rate = rate / nodeCount; if (allocatedCapacity.get() == null) // new thread allocatedCapacity.set(new HashMap<>()); @@ -122,7 +131,7 @@ public class RateLimitingSearcher extends Searcher { requestCapacity(id, rate); } - if (rate==0 || getAllocatedCapacity(id) <= 0) { // we are still over rate: reject + if (rate == 0 || getAllocatedCapacity(id) <= 0) { // we are still over rate: reject String idDim = query.properties().getString(idDimensionKey, null); if (idDim == null) { overQuotaCounter.add(1); diff --git a/container-search/src/main/resources/configdefinitions/search.config.rate-limiting.def b/container-search/src/main/resources/configdefinitions/search.config.rate-limiting.def index e5d2f5f9ed4..81ef06868ce 100644 --- a/container-search/src/main/resources/configdefinitions/search.config.rate-limiting.def +++ b/container-search/src/main/resources/configdefinitions/search.config.rate-limiting.def @@ -17,3 +17,6 @@ maxAvailableCapacity double default=10000 # A good number may be 1 / (maxAvailableCapacity * average-cost) recheckForCapacityProbability double default=0.001 +# Set to true to interpret the rate.quota given in the query as a node-local value +# instead of a cluster-wide value. +localRate bool default=false diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java index 22b8d8f4d76..1682bd37fb6 100755 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java @@ -31,23 +31,10 @@ public class RateLimitingSearcherTestCase { @Test void testRateLimiting() { - RateLimitingConfig.Builder rateLimitingConfig = new RateLimitingConfig.Builder(); - rateLimitingConfig.maxAvailableCapacity(4); - rateLimitingConfig.capacityIncrement(2); - rateLimitingConfig.recheckForCapacityProbability(1.0); - - ClusterInfoConfig.Builder clusterInfoConfig = new ClusterInfoConfig.Builder(); - clusterInfoConfig.clusterId("testCluster"); - clusterInfoConfig.nodeCount(4); - ManualClock clock = new ManualClock(); MetricReceiver.MockReceiver metric = new MetricReceiver.MockReceiver(); - - Chain<Searcher> chain = new Chain<>("test", new RateLimitingSearcher(new RateLimitingConfig(rateLimitingConfig), - new ClusterInfoConfig(clusterInfoConfig), - metric, clock), - new CostSettingSearcher()); - assertEquals(2, tryRequests(chain, "id1"), "'rate' request are available initially"); + var chain = createChain(false, clock, metric); + assertEquals(2, tryRequests(chain, "id1"), "'rate/nodes' request are available initially"); assertTrue(executeWasAllowed(chain, "id1", true), "However, don't reject if we dryRun"); clock.advance(Duration.ofMillis(1500)); // causes 2 new requests to become available assertEquals(2, tryRequests(chain, "id1"), "'rate' new requests became available"); @@ -76,6 +63,34 @@ public class RateLimitingSearcherTestCase { assertEquals(requestsToTry - 2 + requestsToTry - 4, map.get(metric.point("id", "id2")).getCount()); } + @Test + void testLocalRateLimiting() { + ManualClock clock = new ManualClock(); + MetricReceiver.MockReceiver metric = new MetricReceiver.MockReceiver(); + var chain = createChain(true, clock, metric); + + assertEquals(9, tryRequests(chain, "id1"), "'rate' request are available initially"); + } + + private Chain<Searcher> createChain(boolean localRate, ManualClock clock, MetricReceiver.MockReceiver metric) { + RateLimitingConfig.Builder rateLimitingConfig = new RateLimitingConfig.Builder(); + rateLimitingConfig.maxAvailableCapacity(4); + rateLimitingConfig.capacityIncrement(2); + rateLimitingConfig.recheckForCapacityProbability(1.0); + rateLimitingConfig.localRate(localRate); + + ClusterInfoConfig.Builder clusterInfoConfig = new ClusterInfoConfig.Builder(); + clusterInfoConfig.clusterId("testCluster"); + clusterInfoConfig.nodeCount(4); + + + return new Chain<>("test", new RateLimitingSearcher(new RateLimitingConfig(rateLimitingConfig), + new ClusterInfoConfig(clusterInfoConfig), + metric, + clock), + new CostSettingSearcher()); + } + private int requestsToTry = 50; /** diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 4c42349e31f..05daa54f32b 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -85,7 +85,7 @@ <commons-csv.vespa.version>1.10.0</commons-csv.vespa.version> <commons-digester.vespa.version>3.2</commons-digester.vespa.version> <commons-exec.vespa.version>1.3</commons-exec.vespa.version> - <commons-io.vespa.version>2.15.0</commons-io.vespa.version> + <commons-io.vespa.version>2.15.1</commons-io.vespa.version> <commons-lang3.vespa.version>3.14.0</commons-lang3.vespa.version> <commons.math3.vespa.version>3.6.1</commons.math3.vespa.version> <commons-compress.vespa.version>1.25.0</commons-compress.vespa.version> diff --git a/dist/vespa.spec b/dist/vespa.spec index bdae7bdcced..b89d19e5c36 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -501,8 +501,6 @@ fi %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/logs/vespa/search %{_prefix}/man %{_prefix}/sbin -%{_prefix}/share -%exclude %{_prefix}/share/cmake %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/crash %dir %attr(-,%{_vespa_user},%{_vespa_group}) %{_prefix}/var/db @@ -556,6 +554,8 @@ fi %{_prefix}/libexec/vespa/vespa-wrapper %{_prefix}/libexec/vespa/find-pid %{_prefix}/libexec/vespa/vespa-curl-wrapper +%{_prefix}/share +%exclude %{_prefix}/share/cmake %files base-libs %if %{_defattr_is_vespa_vespa} diff --git a/docproc/src/main/java/com/yahoo/docproc/impl/DocprocService.java b/docproc/src/main/java/com/yahoo/docproc/impl/DocprocService.java index b25743fbcde..01df2415ff4 100644 --- a/docproc/src/main/java/com/yahoo/docproc/impl/DocprocService.java +++ b/docproc/src/main/java/com/yahoo/docproc/impl/DocprocService.java @@ -398,7 +398,6 @@ public class DocprocService extends AbstractComponent { } } - private class NoCallStackException extends RuntimeException { - } + private static class NoCallStackException extends RuntimeException { } } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index 4c3292b557b..73d72aabcb9 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -116,7 +116,11 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { @Override protected void destroy() { laterExecutor.shutdown(); - docprocServiceRegistry.allComponents().forEach(docprocService -> docprocService.deconstruct()); + if ( ! laterExecutor.getQueue().isEmpty()) { + // This should not happen, as container should keep this alive until all requests are served. + log.log(Level.SEVERE, "Docproc laterExecutor queue not empty on shutdown, " + laterExecutor.getQueue().size() + " tasks discarded"); + } + docprocServiceRegistry.allComponents().forEach(DocprocService::deconstruct); } public ComponentRegistry<DocprocService> getDocprocServiceRegistry() { @@ -147,7 +151,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { public ContentChannel handleRequest(Request request, ResponseHandler handler) { RequestContext requestContext; if (request instanceof MbusRequest) { - requestContext = new MbusRequestContext((MbusRequest) request, handler, docprocServiceRegistry, docFactoryRegistry, containerDocConfig); + requestContext = new MbusRequestContext((MbusRequest) request, handler, docFactoryRegistry, containerDocConfig); } else { //Other types can be added here in the future throw new IllegalArgumentException("Request type not supported: " + request); diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index 8bcce9e892d..3c6e63bf5e4 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -26,7 +26,7 @@ import java.util.logging.Logger; */ public class DocumentProcessingTask implements Runnable { - private static Logger log = Logger.getLogger(DocumentProcessingTask.class.getName()); + private static final Logger log = Logger.getLogger(DocumentProcessingTask.class.getName()); private final List<Processing> processings = new ArrayList<>(); private final List<Processing> processingsDone = new ArrayList<>(); diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/RequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/RequestContext.java index ce92965e2df..6ec21c507cc 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/RequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/RequestContext.java @@ -28,7 +28,7 @@ public interface RequestContext { void processingFailed(Exception exception); /** Returns whether this request has timed out */ - default boolean hasExpired() { return false;} + default boolean hasExpired() { return false; } void skip(); diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java index e1b9670dea8..bbdabd53084 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java @@ -5,7 +5,6 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.container.core.document.ContainerDocumentConfig; import com.yahoo.docproc.AbstractConcreteDocumentFactory; -import com.yahoo.docproc.impl.DocprocService; import com.yahoo.docproc.impl.HandledProcessingException; import com.yahoo.docproc.Processing; import com.yahoo.docproc.impl.TransientFailureException; @@ -56,11 +55,10 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { private final static String internalNoThrottledSourcePath = "/" + internalNoThrottledSource; public MbusRequestContext(MbusRequest request, ResponseHandler responseHandler, - ComponentRegistry<DocprocService> docprocServiceComponentRegistry, ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, ContainerDocumentConfig containerDocConfig) { this.request = request; - this.requestMsg = (DocumentMessage)request.getMessage(); + this.requestMsg = (DocumentMessage) request.getMessage(); this.responseHandler = responseHandler; this.processingFactory = new ProcessingFactory(docFactoryRegistry, containerDocConfig, getServiceName()); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Response.java b/documentapi/src/main/java/com/yahoo/documentapi/Response.java index eb759e2914a..63c9550474f 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/Response.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/Response.java @@ -105,10 +105,10 @@ public class Response { /** The operation was a success. */ SUCCESS, - /** The operation failed due to an unmet test-and-set condition. */ + /** The operation was not carried out due to an unmet test-and-set condition. */ CONDITION_FAILED, - /** The operation failed because its target document was not found. */ + /** The operation was not carried out because its target document was not found. */ NOT_FOUND, /** The operation failed because the cluster had insufficient storage to accept it. */ diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 4776804a686..2d1792442a0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -290,18 +290,17 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { Message msg = reply.getMessage(); String err = getErrorMessage(reply); Response.Outcome outcome = toOutcome(reply); - switch (msg.getType()) { - case DocumentProtocol.MESSAGE_PUTDOCUMENT: - return new DocumentResponse(reqId, ((PutDocumentMessage)msg).getDocumentPut().getDocument(), err, outcome, reply.getTrace()); - case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: - return new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage)msg).getDocumentUpdate(), err, outcome, reply.getTrace()); - case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: - return new DocumentIdResponse(reqId, ((RemoveDocumentMessage)msg).getDocumentId(), err, outcome, reply.getTrace()); - case DocumentProtocol.MESSAGE_GETDOCUMENT: - return new DocumentIdResponse(reqId, ((GetDocumentMessage)msg).getDocumentId(), err, outcome, reply.getTrace()); - default: - return new Response(reqId, err, outcome, reply.getTrace()); - } + return switch (msg.getType()) { + case DocumentProtocol.MESSAGE_PUTDOCUMENT -> + new DocumentResponse(reqId, ((PutDocumentMessage) msg).getDocumentPut().getDocument(), err, outcome, reply.getTrace()); + case DocumentProtocol.MESSAGE_UPDATEDOCUMENT -> + new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage) msg).getDocumentUpdate(), err, outcome, reply.getTrace()); + case DocumentProtocol.MESSAGE_REMOVEDOCUMENT -> + new DocumentIdResponse(reqId, ((RemoveDocumentMessage) msg).getDocumentId(), err, outcome, reply.getTrace()); + case DocumentProtocol.MESSAGE_GETDOCUMENT -> + new DocumentIdResponse(reqId, ((GetDocumentMessage) msg).getDocumentId(), err, outcome, reply.getTrace()); + default -> new Response(reqId, err, outcome, reply.getTrace()); + }; } private static Response toSuccess(Reply reply, long reqId) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java index 03932d1b69e..5af78d97509 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java @@ -595,13 +595,11 @@ public class ContentPolicy extends SlobrokPolicy { return !reply.hasErrors(); // For simplicity, count any reply with > 1 error. } var error = reply.getError(0); - switch (error.getCode()) { + return switch (error.getCode()) { // TODO this feels like a layering violation, but we use DocumentProtocol directly in other places in this policy anyway... - case DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED: - case DocumentProtocol.ERROR_BUSY: - return false; - default: return true; - } + case DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED, DocumentProtocol.ERROR_BUSY -> false; + default -> true; + }; } void handleErrorReply(Reply reply, Object untypedContext) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index b53446e9d39..ec49a0c570f 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -157,7 +157,7 @@ public class DocumentProtocol implements Protocol { /** We failed accessing the disk, which we think is a disk hardware problem. */ public static final int ERROR_DISK_FAILURE = ErrorCode.APP_TRANSIENT_ERROR + 1007; - /** We failed during an IO operation, we dont think is a specific disk hardware problem. */ + /** We failed during an IO operation, we don't think is a specific disk hardware problem. */ public static final int ERROR_IO_FAILURE = ErrorCode.APP_TRANSIENT_ERROR + 1008; /** diff --git a/eval/src/apps/eval_expr/eval_expr.cpp b/eval/src/apps/eval_expr/eval_expr.cpp index 23af5a926c4..af7dd26ca26 100644 --- a/eval/src/apps/eval_expr/eval_expr.cpp +++ b/eval/src/apps/eval_expr/eval_expr.cpp @@ -406,6 +406,7 @@ int main(int argc, char **argv) { } Context ctx; if ((expr_cnt == 1) && (vespalib::string(argv[expr_idx]) == "interactive")) { + setlocale(LC_ALL, ""); return interactive_mode(ctx); } if ((expr_cnt == 1) && (vespalib::string(argv[expr_idx]) == "json-repl")) { diff --git a/fbench/src/splitfile/splitfile.cpp b/fbench/src/splitfile/splitfile.cpp index 65eb0009564..8eddc163463 100644 --- a/fbench/src/splitfile/splitfile.cpp +++ b/fbench/src/splitfile/splitfile.cpp @@ -28,6 +28,10 @@ main(int argc, char** argv) switch(opt) { case 'p': pattern = optarg; + if (pattern == nullptr) { + printf("Missing 'pattern' argument to -p option !\n"); + return -1; + } break; case 'm': linebufsize = atoi(optarg); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Dimension.java b/flags/src/main/java/com/yahoo/vespa/flags/Dimension.java new file mode 100644 index 00000000000..0f81fd4640b --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/Dimension.java @@ -0,0 +1,120 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Flag dimensions. + * + * <ol> + * <li>A flag definition declares the dimensions it supports.*</li> + * <li>To <em>get</em> the value of a flag, a {@link FetchVector} should be built with the same set of dimensions.*</li> + * <li>To <em>set</em> the value of a flag, add a flag data <em>rule</em> with that value. The rule may be + * contain conditions that refer to these dimension.</li> + * </ol> + * + * <p>*) The system, cloud, environment, and zone dimensions are special: A flag should NOT list them in the + * flag definition (1),** and the dimensions are automatically set (2),**. These dimensions may always be referred to + * when overriding values, either as dimensions (3) or in dedicated files.</p> + * <p>**) The controller may want different flag values depending on cloud, environment, and/or zone. + * Disregard (*) for those dimensions: The relevant dimensions should be declared in the flag definition (1), + * and specified when getting the value (2).</p> + * + * @author hakonhall + */ +public enum Dimension { + /** + * Application from ApplicationId::toSerializedFormWithoutInstance() of the form tenant:applicationName. + * <p><em>WARNING: NOT ApplicationId</em> - see {@link #INSTANCE_ID}.</p> + */ + APPLICATION("application"), + + /** Machine architecture: either arm64 or x86_64. */ + ARCHITECTURE("architecture"), + + /** Whether "enclave" (or "inclave" or "exclave"), or not ("noclave"). */ + CLAVE("clave"), + + /** + * Cloud from com.yahoo.config.provision.CloudName::value, e.g. yahoo, aws, gcp. + * + * <p><em>Eager resolution</em>: This dimension is resolved before putting the flag data to the config server + * or controller, unless controller and the flag has declared this dimension. + */ + CLOUD("cloud"), + + /** Cloud account ID from com.yahoo.config.provision.CloudAccount::value, e.g. aws:123456789012 */ + CLOUD_ACCOUNT("cloud-account"), + + /** Cluster ID from com.yahoo.config.provision.ClusterSpec.Id::value, e.g. cluster-controllers, logserver. */ + CLUSTER_ID("cluster-id"), + + /** Cluster type from com.yahoo.config.provision.ClusterSpec.Type::name, e.g. content, container, admin */ + CLUSTER_TYPE("cluster-type"), + + /** Email address of user - provided by auth0 in console. */ + CONSOLE_USER_EMAIL("console-user-email"), + + /** Hosted Vespa environment from com.yahoo.config.provision.Environment::value, e.g. prod, staging, test. */ + ENVIRONMENT("environment"), + + /** + * Fully qualified hostname. + * + * <p>NOTE: There is seldom any need to set HOSTNAME, as it is always set implicitly (in {@link Flags}) + * from {@code Defaults.getDefaults().vespaHostname()}. The hostname may e.g. be overridden when + * fetching flag value for a Docker container node. + */ + HOSTNAME("hostname"), + + /** Value from ApplicationId::serializedForm of the form tenant:applicationName:instance. */ + INSTANCE_ID("instance"), + + /** Node type from com.yahoo.config.provision.NodeType::name, e.g. tenant, host, confighost, controller, etc. */ + NODE_TYPE("node-type"), + + /** + * Hosted Vespa system from com.yahoo.config.provision.SystemName::value, e.g. main, cd, public, publiccd. + * <em>Eager resolution</em>, see {@link #CLOUD}. + */ + SYSTEM("system"), + + /** Value from TenantName::value, e.g. vespa-team */ + TENANT_ID("tenant"), + + /** + * Vespa version from Version::toFullString of the form Major.Minor.Micro. + * + * <p>NOTE: There is seldom any need to set VESPA_VERSION, as it is always set implicitly + * (in {@link Flags}) from {@link com.yahoo.component.Vtag#currentVersion}. The version COULD e.g. + * be overridden when fetching flag value for a Docker container node. + */ + VESPA_VERSION("vespa-version"), + + /** + * Virtual zone ID from com.yahoo.config.provision.zone.ZoneId::value of the form environment.region, + * see com.yahoo.config.provision.zone.ZoneApi::getVirtualId. <em>Eager resolution</em>, see {@link #CLOUD}. + */ + ZONE_ID("zone"); + + private final String wireName; + + private static final Map<String, Dimension> dimensionsByWireName = + Stream.of(values()).collect(Collectors.toMap(x -> x.wireName, Function.identity())); + + public static Dimension fromWire(String wireName) { + Dimension dimension = dimensionsByWireName.get(wireName); + if (dimension == null) { + throw new IllegalArgumentException("Unknown serialized dimension: '" + wireName + "'"); + } + + return dimension; + } + + Dimension(String wireName) { this.wireName = wireName; } + + public String toWire() { return wireName; } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index b5a944430f3..5639cbc0143 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -1,8 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import com.yahoo.vespa.flags.json.DimensionHelper; - import java.util.Collection; import java.util.EnumMap; import java.util.Map; @@ -18,80 +16,6 @@ import java.util.function.Consumer; * @author hakonhall */ public class FetchVector { - /** - * Note: If this enum is changed, you must also change {@link DimensionHelper}. - */ - public enum Dimension { - /** - * Application from ApplicationId::toSerializedFormWithoutInstance() of the form tenant:applicationName. - * <p><em>WARNING: NOT ApplicationId</em> - see {@link #INSTANCE_ID}.</p> - */ - APPLICATION, - - /** - * Cloud from com.yahoo.config.provision.CloudName::value, e.g. yahoo, aws, gcp. - * - * <p><em>Eager resolution</em>: This dimension is resolved before putting the flag data to the config server - * or controller, unless controller and the flag has declared this dimension. - */ - CLOUD, - - /** - * Cloud account ID from com.yahoo.config.provision.CloudAccount::value, e.g. aws:123456789012 - */ - CLOUD_ACCOUNT, - - /** Cluster ID from com.yahoo.config.provision.ClusterSpec.Id::value, e.g. cluster-controllers, logserver. */ - CLUSTER_ID, - - /** Cluster type from com.yahoo.config.provision.ClusterSpec.Type::name, e.g. content, container, admin */ - CLUSTER_TYPE, - - /** Email address of user - provided by auth0 in console. */ - CONSOLE_USER_EMAIL, - - /** Hosted Vespa environment from com.yahoo.config.provision.Environment::value, e.g. prod, staging, test. */ - ENVIRONMENT, - - /** - * Fully qualified hostname. - * - * <p>NOTE: There is seldom any need to set HOSTNAME, as it is always set implicitly (in {@link Flags}) - * from {@code Defaults.getDefaults().vespaHostname()}. The hostname may e.g. be overridden when - * fetching flag value for a Docker container node. - */ - HOSTNAME, - - /** Value from ApplicationId::serializedForm of the form tenant:applicationName:instance. */ - INSTANCE_ID, - - /** Node type from com.yahoo.config.provision.NodeType::name, e.g. tenant, host, confighost, controller, etc. */ - NODE_TYPE, - - /** - * Hosted Vespa system from com.yahoo.config.provision.SystemName::value, e.g. main, cd, public, publiccd. - * <em>Eager resolution</em>, see {@link #CLOUD}. - */ - SYSTEM, - - /** Value from TenantName::value, e.g. vespa-team */ - TENANT_ID, - - /** - * Vespa version from Version::toFullString of the form Major.Minor.Micro. - * - * <p>NOTE: There is seldom any need to set VESPA_VERSION, as it is always set implicitly - * (in {@link Flags}) from {@link com.yahoo.component.Vtag#currentVersion}. The version COULD e.g. - * be overridden when fetching flag value for a Docker container node. - */ - VESPA_VERSION, - - /** - * Virtual zone ID from com.yahoo.config.provision.zone.ZoneId::value of the form environment.region, - * see com.yahoo.config.provision.zone.ZoneApi::getVirtualId. <em>Eager resolution</em>, see {@link #CLOUD}. - */ - ZONE_ID - } private final Map<Dimension, String> map; @@ -115,7 +39,7 @@ public class FetchVector { public boolean isEmpty() { return map.isEmpty(); } - public boolean hasDimension(FetchVector.Dimension dimension) { return map.containsKey(dimension);} + public boolean hasDimension(Dimension dimension) { return map.containsKey(dimension);} public Set<Dimension> dimensions() { return map.keySet(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flag.java b/flags/src/main/java/com/yahoo/vespa/flags/Flag.java index 0ca9dbb4cf7..7ca5066969f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flag.java @@ -21,10 +21,10 @@ public interface Flag<T, F> { FlagSerializer<T> serializer(); /** Returns an immutable clone of the current object, except with the dimension set accordingly. */ - F with(FetchVector.Dimension dimension, String dimensionValue); + F with(Dimension dimension, String dimensionValue); - /** Same as {@link #with(FetchVector.Dimension, String)} if value is present, and otherwise returns {@code this}. */ - default F with(FetchVector.Dimension dimension, Optional<String> dimensionValue) { + /** Same as {@link #with(Dimension, String)} if value is present, and otherwise returns {@code this}. */ + default F with(Dimension dimension, Optional<String> dimensionValue) { return dimensionValue.map(value -> with(dimension, value)).orElse(self()); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java index 837bc3b6e11..181c7ebd066 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java @@ -16,7 +16,7 @@ public class FlagDefinition { private final Instant expiresAt; private final String description; private final String modificationEffect; - private final List<FetchVector.Dimension> dimensions; + private final List<Dimension> dimensions; public FlagDefinition( UnboundFlag<?, ?, ?> unboundFlag, @@ -25,7 +25,7 @@ public class FlagDefinition { Instant expiresAt, String description, String modificationEffect, - FetchVector.Dimension... dimensions) { + Dimension... dimensions) { this.unboundFlag = unboundFlag; this.owners = owners; this.createdAt = createdAt; @@ -40,7 +40,7 @@ public class FlagDefinition { return unboundFlag; } - public List<FetchVector.Dimension> getDimensions() { + public List<Dimension> getDimensions() { return dimensions; } @@ -58,7 +58,7 @@ public class FlagDefinition { public Instant getExpiresAt() { return expiresAt; } - private static void validate(List<String> owners, Instant createdAt, Instant expiresAt, List<FetchVector.Dimension> dimensions) { + private static void validate(List<String> owners, Instant createdAt, Instant expiresAt, List<Dimension> dimensions) { if (expiresAt.isBefore(createdAt)) { throw new IllegalArgumentException( String.format( @@ -74,14 +74,14 @@ public class FlagDefinition { throw new IllegalArgumentException("Owner(s) must be specified"); } - if (dimensions.contains(FetchVector.Dimension.CONSOLE_USER_EMAIL)) { - Set<FetchVector.Dimension> disallowedCombinations = EnumSet.allOf(FetchVector.Dimension.class); - disallowedCombinations.remove(FetchVector.Dimension.CONSOLE_USER_EMAIL); - disallowedCombinations.remove(FetchVector.Dimension.INSTANCE_ID); - disallowedCombinations.remove(FetchVector.Dimension.TENANT_ID); + if (dimensions.contains(Dimension.CONSOLE_USER_EMAIL)) { + Set<Dimension> disallowedCombinations = EnumSet.allOf(Dimension.class); + disallowedCombinations.remove(Dimension.CONSOLE_USER_EMAIL); + disallowedCombinations.remove(Dimension.INSTANCE_ID); + disallowedCombinations.remove(Dimension.TENANT_ID); disallowedCombinations.retainAll(dimensions); if (!disallowedCombinations.isEmpty()) - throw new IllegalArgumentException("Dimension " + FetchVector.Dimension.CONSOLE_USER_EMAIL + " cannot be combined with " + disallowedCombinations); + throw new IllegalArgumentException("Dimension " + Dimension.CONSOLE_USER_EMAIL + " cannot be combined with " + disallowedCombinations); } } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagImpl.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagImpl.java index 1b0464be7c6..496217c1e58 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FlagImpl.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagImpl.java @@ -31,7 +31,7 @@ public abstract class FlagImpl<T, F extends FlagImpl<T, F>> implements Flag<T, F } @Override - public F with(FetchVector.Dimension dimension, String dimensionValue) { + public F with(Dimension dimension, String dimensionValue) { return factory.create(id, defaultValue, fetchVector.with(dimension, dimensionValue), serializer, source); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index ca5dd6491b3..7d62826a3a2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -13,14 +13,16 @@ import java.util.Optional; import java.util.TreeMap; import java.util.function.Predicate; -import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CLOUD_ACCOUNT; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CONSOLE_USER_EMAIL; -import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; -import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID; -import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; -import static com.yahoo.vespa.flags.FetchVector.Dimension.TENANT_ID; -import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; +import static com.yahoo.vespa.flags.Dimension.APPLICATION; +import static com.yahoo.vespa.flags.Dimension.ARCHITECTURE; +import static com.yahoo.vespa.flags.Dimension.CLAVE; +import static com.yahoo.vespa.flags.Dimension.CLOUD_ACCOUNT; +import static com.yahoo.vespa.flags.Dimension.CONSOLE_USER_EMAIL; +import static com.yahoo.vespa.flags.Dimension.HOSTNAME; +import static com.yahoo.vespa.flags.Dimension.INSTANCE_ID; +import static com.yahoo.vespa.flags.Dimension.NODE_TYPE; +import static com.yahoo.vespa.flags.Dimension.TENANT_ID; +import static com.yahoo.vespa.flags.Dimension.VESPA_VERSION; /** * Definitions of feature flags. @@ -34,8 +36,8 @@ import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; * an unbound flag to a flag source produces a (bound) flag, e.g. {@link BooleanFlag} and {@link StringFlag}.</li> * <li>If you would like your flag value to be dependent on e.g. the application ID, then 1. you should * declare this in the unbound flag definition in this file (referring to - * {@link FetchVector.Dimension#INSTANCE_ID}), and 2. specify the application ID when retrieving the value, e.g. - * {@link BooleanFlag#with(FetchVector.Dimension, String)}. See {@link FetchVector} for more info.</li> + * {@link Dimension#INSTANCE_ID}), and 2. specify the application ID when retrieving the value, e.g. + * {@link BooleanFlag#with(Dimension, String)}. See {@link FetchVector} for more info.</li> * </ol> * * <p>Once the code is in place, you can override the flag value. This depends on the flag source, but typically @@ -76,6 +78,15 @@ public class Flags { "Takes effect at redeployment (requires restart)", INSTANCE_ID); + public static final UnboundStringFlag NESSUS_AGENT_GROUP = defineStringFlag( + "nessus-agent-group", "All", + List.of("hakonhall"), "2023-11-29", "2023-12-29", + "Either run nessusagent as before (All), or link against \"vespa-ci\"," + + " or disable the nessusagent (empty string \"\")", + "Takes effect after host admin restart", + (String value) -> value.equals("All") || value.equals("vespa-ci") || value.isEmpty(), + ARCHITECTURE, CLAVE); + public static final UnboundIntFlag MAX_UNCOMMITTED_MEMORY = defineIntFlag( "max-uncommitted-memory", 130000, List.of("geirst, baldersheim"), "2021-10-21", "2023-12-31", @@ -407,17 +418,24 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); + public static final UnboundBooleanFlag ENABLE_NEW_PAYMENT_METHOD_FLOW = defineFeatureFlag( + "enable-new-payment-method-flow", false, + List.of("bjorncs"), "2023-11-29", "2024-03-01", + "Whether to enable the new billing flow", + "Takes effect immediately", + TENANT_ID, CONSOLE_USER_EMAIL); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, - String modificationEffect, FetchVector.Dimension... dimensions) { + String modificationEffect, Dimension... dimensions) { return define(UnboundBooleanFlag::new, flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, dimensions); } /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundStringFlag defineStringFlag(String flagId, String defaultValue, List<String> owners, String createdAt, String expiresAt, String description, - String modificationEffect, FetchVector.Dimension... dimensions) { + String modificationEffect, Dimension... dimensions) { return defineStringFlag(flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, value -> true, @@ -428,7 +446,7 @@ public class Flags { public static UnboundStringFlag defineStringFlag(String flagId, String defaultValue, List<String> owners, String createdAt, String expiresAt, String description, String modificationEffect, Predicate<String> validator, - FetchVector.Dimension... dimensions) { + Dimension... dimensions) { return define((i, d, v) -> new UnboundStringFlag(i, d, v, validator), flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, dimensions); } @@ -436,28 +454,28 @@ public class Flags { /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundIntFlag defineIntFlag(String flagId, int defaultValue, List<String> owners, String createdAt, String expiresAt, String description, - String modificationEffect, FetchVector.Dimension... dimensions) { + String modificationEffect, Dimension... dimensions) { return define(UnboundIntFlag::new, flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, dimensions); } /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundLongFlag defineLongFlag(String flagId, long defaultValue, List<String> owners, String createdAt, String expiresAt, String description, - String modificationEffect, FetchVector.Dimension... dimensions) { + String modificationEffect, Dimension... dimensions) { return define(UnboundLongFlag::new, flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, dimensions); } /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundDoubleFlag defineDoubleFlag(String flagId, double defaultValue, List<String> owners, String createdAt, String expiresAt, String description, - String modificationEffect, FetchVector.Dimension... dimensions) { + String modificationEffect, Dimension... dimensions) { return define(UnboundDoubleFlag::new, flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, dimensions); } /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static <T> UnboundJacksonFlag<T> defineJacksonFlag(String flagId, T defaultValue, Class<T> jacksonClass, List<String> owners, String createdAt, String expiresAt, String description, - String modificationEffect, FetchVector.Dimension... dimensions) { + String modificationEffect, Dimension... dimensions) { return define((id2, defaultValue2, vector2) -> new UnboundJacksonFlag<>(id2, defaultValue2, vector2, jacksonClass), flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, dimensions); } @@ -465,7 +483,7 @@ public class Flags { /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static <T> UnboundListFlag<T> defineListFlag(String flagId, List<T> defaultValue, Class<T> elementClass, List<String> owners, String createdAt, String expiresAt, - String description, String modificationEffect, FetchVector.Dimension... dimensions) { + String description, String modificationEffect, Dimension... dimensions) { return define((fid, dval, fvec) -> new UnboundListFlag<>(fid, dval, elementClass, fvec), flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, dimensions); } @@ -500,8 +518,8 @@ public class Flags { * use them in the controller in this way. * @param <T> The boxed type of the flag value, e.g. Boolean for flags guarding features. * @param <U> The type of the unbound flag, e.g. UnboundBooleanFlag. - * @return An unbound flag with {@link FetchVector.Dimension#HOSTNAME HOSTNAME} and - * {@link FetchVector.Dimension#VESPA_VERSION VESPA_VERSION} already set. The ZONE environment + * @return An unbound flag with {@link Dimension#HOSTNAME HOSTNAME} and + * {@link Dimension#VESPA_VERSION VESPA_VERSION} already set. The ZONE environment * is typically implicit. */ private static <T, U extends UnboundFlag<?, ?, ?>> U define(TypedUnboundFlagFactory<T, U> factory, @@ -512,7 +530,7 @@ public class Flags { String expiresAt, String description, String modificationEffect, - FetchVector.Dimension[] dimensions) { + Dimension[] dimensions) { FlagId id = new FlagId(flagId); FetchVector vector = new FetchVector() .with(HOSTNAME, Defaults.getDefaults().vespaHostname()) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 1c50cc54964..acebca05fdb 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -13,15 +13,15 @@ import java.util.Set; import java.util.function.Predicate; import java.util.regex.Pattern; -import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_ID; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CONSOLE_USER_EMAIL; -import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; -import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; -import static com.yahoo.vespa.flags.FetchVector.Dimension.TENANT_ID; -import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; -import static com.yahoo.vespa.flags.FetchVector.Dimension.ZONE_ID; +import static com.yahoo.vespa.flags.Dimension.INSTANCE_ID; +import static com.yahoo.vespa.flags.Dimension.CLUSTER_ID; +import static com.yahoo.vespa.flags.Dimension.CLUSTER_TYPE; +import static com.yahoo.vespa.flags.Dimension.CONSOLE_USER_EMAIL; +import static com.yahoo.vespa.flags.Dimension.HOSTNAME; +import static com.yahoo.vespa.flags.Dimension.NODE_TYPE; +import static com.yahoo.vespa.flags.Dimension.TENANT_ID; +import static com.yahoo.vespa.flags.Dimension.VESPA_VERSION; +import static com.yahoo.vespa.flags.Dimension.ZONE_ID; /** * Definition for permanent feature flags @@ -417,42 +417,42 @@ public class PermanentFlags { private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( - String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { + String flagId, boolean defaultValue, String description, String modificationEffect, Dimension... dimensions) { return Flags.defineFeatureFlag(flagId, defaultValue, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, dimensions); } private static UnboundStringFlag defineStringFlag( - String flagId, String defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { + String flagId, String defaultValue, String description, String modificationEffect, Dimension... dimensions) { return Flags.defineStringFlag(flagId, defaultValue, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, dimensions); } private static UnboundStringFlag defineStringFlag( - String flagId, String defaultValue, String description, String modificationEffect, Predicate<String> validator, FetchVector.Dimension... dimensions) { + String flagId, String defaultValue, String description, String modificationEffect, Predicate<String> validator, Dimension... dimensions) { return Flags.defineStringFlag(flagId, defaultValue, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, validator, dimensions); } private static UnboundIntFlag defineIntFlag( - String flagId, int defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { + String flagId, int defaultValue, String description, String modificationEffect, Dimension... dimensions) { return Flags.defineIntFlag(flagId, defaultValue, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, dimensions); } private static UnboundLongFlag defineLongFlag( - String flagId, long defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { + String flagId, long defaultValue, String description, String modificationEffect, Dimension... dimensions) { return Flags.defineLongFlag(flagId, defaultValue, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, dimensions); } private static UnboundDoubleFlag defineDoubleFlag( - String flagId, double defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { + String flagId, double defaultValue, String description, String modificationEffect, Dimension... dimensions) { return Flags.defineDoubleFlag(flagId, defaultValue, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, dimensions); } private static <T> UnboundJacksonFlag<T> defineJacksonFlag( - String flagId, T defaultValue, Class<T> jacksonClass, String description, String modificationEffect, FetchVector.Dimension... dimensions) { + String flagId, T defaultValue, Class<T> jacksonClass, String description, String modificationEffect, Dimension... dimensions) { return Flags.defineJacksonFlag(flagId, defaultValue, jacksonClass, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, dimensions); } private static <T> UnboundListFlag<T> defineListFlag( - String flagId, List<T> defaultValue, Class<T> elementClass, String description, String modificationEffect, FetchVector.Dimension... dimensions) { + String flagId, List<T> defaultValue, Class<T> elementClass, String description, String modificationEffect, Dimension... dimensions) { return Flags.defineListFlag(flagId, defaultValue, elementClass, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, dimensions); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundFlag.java index edde650adc9..f6d4e080e82 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/UnboundFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundFlag.java @@ -18,7 +18,7 @@ public interface UnboundFlag<T, F extends Flag<T, F>, U extends UnboundFlag<T, F FlagSerializer<T> serializer(); /** Returns a clone of the unbound flag, but with the dimension set accordingly. */ - U with(FetchVector.Dimension dimension, String dimensionValue); + U with(Dimension dimension, String dimensionValue); /** Binds to a flag source, returning a (bound) flag. */ F bindTo(FlagSource source); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundFlagImpl.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundFlagImpl.java index a079bf734d7..d8936e17395 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/UnboundFlagImpl.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundFlagImpl.java @@ -42,7 +42,7 @@ public abstract class UnboundFlagImpl<T, F extends Flag<T, F>, U extends Unbound } @Override - public U with(FetchVector.Dimension dimension, String dimensionValue) { + public U with(Dimension dimension, String dimensionValue) { return unboundFlagFactory.create(id, defaultValue, defaultFetchVector.with(dimension, dimensionValue)); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java index 2881e9ab9ad..247987e68f4 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/Condition.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags.json; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.json.wire.WireCondition; @@ -32,11 +33,11 @@ public interface Condition extends Predicate<FetchVector> { } class CreateParams { - private final FetchVector.Dimension dimension; + private final Dimension dimension; private List<String> values = List.of(); private Optional<String> predicate = Optional.empty(); - public CreateParams(FetchVector.Dimension dimension) { this.dimension = Objects.requireNonNull(dimension); } + public CreateParams(Dimension dimension) { this.dimension = Objects.requireNonNull(dimension); } public CreateParams withValues(String... values) { return withValues(List.of(values)); } public CreateParams withValues(List<String> values) { @@ -49,7 +50,7 @@ public interface Condition extends Predicate<FetchVector> { return this; } - public FetchVector.Dimension dimension() { return dimension; } + public Dimension dimension() { return dimension; } public List<String> values() { return values; } public Optional<String> predicate() { return predicate; } @@ -69,7 +70,7 @@ public interface Condition extends Predicate<FetchVector> { Condition.Type type = Condition.Type.fromWire(wireCondition.type); Objects.requireNonNull(wireCondition.dimension); - FetchVector.Dimension dimension = DimensionHelper.fromWire(wireCondition.dimension); + Dimension dimension = Dimension.fromWire(wireCondition.dimension); var params = new CreateParams(dimension); if (wireCondition.values != null) { @@ -85,7 +86,7 @@ public interface Condition extends Predicate<FetchVector> { Condition.Type type(); - FetchVector.Dimension dimension(); + Dimension dimension(); CreateParams toCreateParams(); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java deleted file mode 100644 index 7298f090be2..00000000000 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.flags.json; - -import com.yahoo.vespa.flags.FetchVector; - -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * @author hakonhall - */ -public class DimensionHelper { - - private static final Map<FetchVector.Dimension, String> serializedDimensions = new HashMap<>(); - - static { - // WARNING: If you ever change the serialized form of a dimension, ensure the new serialized - // flag data are pushed out everywhere before removing support for old format, see VESPA-27760. - serializedDimensions.put(FetchVector.Dimension.APPLICATION, "application"); - serializedDimensions.put(FetchVector.Dimension.CLOUD, "cloud"); - serializedDimensions.put(FetchVector.Dimension.CLOUD_ACCOUNT, "cloud-account"); - serializedDimensions.put(FetchVector.Dimension.CLUSTER_ID, "cluster-id"); - serializedDimensions.put(FetchVector.Dimension.CLUSTER_TYPE, "cluster-type"); - serializedDimensions.put(FetchVector.Dimension.CONSOLE_USER_EMAIL, "console-user-email"); - serializedDimensions.put(FetchVector.Dimension.ENVIRONMENT, "environment"); - serializedDimensions.put(FetchVector.Dimension.HOSTNAME, "hostname"); - serializedDimensions.put(FetchVector.Dimension.INSTANCE_ID, "instance"); - serializedDimensions.put(FetchVector.Dimension.NODE_TYPE, "node-type"); - serializedDimensions.put(FetchVector.Dimension.SYSTEM, "system"); - serializedDimensions.put(FetchVector.Dimension.TENANT_ID, "tenant"); - serializedDimensions.put(FetchVector.Dimension.VESPA_VERSION, "vespa-version"); - serializedDimensions.put(FetchVector.Dimension.ZONE_ID, "zone"); - - if (serializedDimensions.size() != FetchVector.Dimension.values().length) { - throw new IllegalStateException(FetchVectorHelper.class.getName() + " is not in sync with " + - FetchVector.Dimension.class.getName()); - } - } - - private static final Map<String, FetchVector.Dimension> deserializedDimensions = serializedDimensions. - entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); - - public static String toWire(FetchVector.Dimension dimension) { - String serializedDimension = serializedDimensions.get(dimension); - if (serializedDimension == null) { - throw new IllegalArgumentException("Unsupported dimension (please add it): '" + dimension + "'"); - } - - return serializedDimension; - } - - public static FetchVector.Dimension fromWire(String serializedDimension) { - FetchVector.Dimension dimension = deserializedDimensions.get(serializedDimension); - if (dimension == null) { - throw new IllegalArgumentException("Unknown serialized dimension: '" + serializedDimension + "'"); - } - - return dimension; - } - - private DimensionHelper() { } - -} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FetchVectorHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FetchVectorHelper.java index 31ba0e1e6cf..97ea4695481 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/FetchVectorHelper.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FetchVectorHelper.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags.json; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.FetchVector; import java.util.Map; @@ -12,17 +13,17 @@ import java.util.stream.Collectors; public class FetchVectorHelper { public static Map<String, String> toWire(FetchVector vector) { - Map<FetchVector.Dimension, String> map = vector.toMap(); + Map<Dimension, String> map = vector.toMap(); if (map.isEmpty()) return null; return map.entrySet().stream().collect(Collectors.toMap( - entry -> DimensionHelper.toWire(entry.getKey()), + entry -> entry.getKey().toWire(), Map.Entry::getValue)); } public static FetchVector fromWire(Map<String, String> wireMap) { if (wireMap == null) return new FetchVector(); return FetchVector.fromMap(wireMap.entrySet().stream().collect(Collectors.toMap( - entry -> DimensionHelper.fromWire(entry.getKey()), + entry -> Dimension.fromWire(entry.getKey()), Map.Entry::getValue))); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java index 12a10298787..7e71bb10a46 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/ListCondition.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags.json; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.json.wire.WireCondition; @@ -12,7 +13,7 @@ import java.util.Objects; */ public abstract class ListCondition implements Condition { private final Condition.Type type; - private final FetchVector.Dimension dimension; + private final Dimension dimension; private final List<String> values; private final boolean isWhitelist; @@ -33,7 +34,7 @@ public abstract class ListCondition implements Condition { } @Override - public FetchVector.Dimension dimension() { + public Dimension dimension() { return dimension; } @@ -52,7 +53,7 @@ public abstract class ListCondition implements Condition { public WireCondition toWire() { var condition = new WireCondition(); condition.type = type.toWire(); - condition.dimension = DimensionHelper.toWire(dimension); + condition.dimension = dimension.toWire(); condition.values = values.isEmpty() ? null : values; return condition; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java index 49dc7c75752..18d1052d43c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.flags.json; import com.yahoo.component.Version; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.json.wire.WireCondition; @@ -14,7 +15,7 @@ import java.util.function.Predicate; public class RelationalCondition implements Condition { private final RelationalPredicate relationalPredicate; private final Predicate<String> predicate; - private final FetchVector.Dimension dimension; + private final Dimension dimension; public static RelationalCondition create(CreateParams params) { if (!params.values().isEmpty()) { @@ -37,12 +38,12 @@ public class RelationalCondition implements Condition { return new RelationalCondition(relationalPredicate, p, params.dimension()); default: throw new IllegalArgumentException(RelationalCondition.class.getSimpleName() + - " not supported for dimension " + FetchVector.Dimension.VESPA_VERSION.name()); + " not supported for dimension " + Dimension.VESPA_VERSION.name()); } } private RelationalCondition(RelationalPredicate relationalPredicate, Predicate<String> predicate, - FetchVector.Dimension dimension) { + Dimension dimension) { this.relationalPredicate = relationalPredicate; this.predicate = predicate; this.dimension = dimension; @@ -54,7 +55,7 @@ public class RelationalCondition implements Condition { } @Override - public FetchVector.Dimension dimension() { + public Dimension dimension() { return dimension; } @@ -76,7 +77,7 @@ public class RelationalCondition implements Condition { public WireCondition toWire() { var condition = new WireCondition(); condition.type = Condition.Type.RELATIONAL.toWire(); - condition.dimension = DimensionHelper.toWire(dimension); + condition.dimension = dimension.toWire(); condition.predicate = relationalPredicate.toWire(); return condition; } diff --git a/flags/src/test/java/com/yahoo/vespa/flags/DimensionTest.java b/flags/src/test/java/com/yahoo/vespa/flags/DimensionTest.java new file mode 100644 index 00000000000..11687265a05 --- /dev/null +++ b/flags/src/test/java/com/yahoo/vespa/flags/DimensionTest.java @@ -0,0 +1,25 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; + +/** + * @author hakonhall + */ +class DimensionTest { + /** + * A compile-time test: If this breaks you have most likely added (or removed) a dimension? + * If so you need to update the following:</p> + * + * <ul> + * <li>Dimension validation in SystemFlagsDataArchive</li> + * <li>Flag edit documenting dimension list in EditFlagDataCommandTest</li> + * </ul> + */ + @SuppressWarnings("unused") + public String remember_to_update_SystemFlagsDataArchive(Dimension dimension) { + return switch (dimension) { + case APPLICATION, ARCHITECTURE, CLAVE, CLOUD, CLOUD_ACCOUNT, CLUSTER_ID, CLUSTER_TYPE, + CONSOLE_USER_EMAIL, ENVIRONMENT, HOSTNAME, INSTANCE_ID, NODE_TYPE, SYSTEM, TENANT_ID, + VESPA_VERSION, ZONE_ID -> dimension.toWire(); + }; + } +}
\ No newline at end of file diff --git a/flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java b/flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java index d3a5406bead..d48e8ec9c8b 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/FlagsTest.java @@ -32,8 +32,8 @@ public class FlagsTest { final boolean defaultValue = false; FlagSource source = mock(FlagSource.class); BooleanFlag booleanFlag = Flags.defineFeatureFlag("id", defaultValue, List.of("owner"), "1970-01-01", "2100-01-01", "description", - "modification effect", FetchVector.Dimension.ZONE_ID, FetchVector.Dimension.HOSTNAME) - .with(FetchVector.Dimension.ZONE_ID, "a-zone") + "modification effect", Dimension.ZONE_ID, Dimension.HOSTNAME) + .with(Dimension.ZONE_ID, "a-zone") .bindTo(source); assertThat(booleanFlag.id().toString(), equalTo("id")); @@ -44,31 +44,31 @@ public class FlagsTest { ArgumentCaptor<FetchVector> vector = ArgumentCaptor.forClass(FetchVector.class); verify(source).fetch(any(), vector.capture()); // hostname is set by default - Optional<String> hostname = vector.getValue().getValue(FetchVector.Dimension.HOSTNAME); + Optional<String> hostname = vector.getValue().getValue(Dimension.HOSTNAME); assertTrue(hostname.isPresent()); assertFalse(hostname.get().isEmpty()); // zone is set because it was set on the unbound flag above - assertThat(vector.getValue().getValue(FetchVector.Dimension.ZONE_ID), is(Optional.of("a-zone"))); + assertThat(vector.getValue().getValue(Dimension.ZONE_ID), is(Optional.of("a-zone"))); // application and node type are not set - assertThat(vector.getValue().getValue(FetchVector.Dimension.INSTANCE_ID), is(Optional.empty())); - assertThat(vector.getValue().getValue(FetchVector.Dimension.NODE_TYPE), is(Optional.empty())); + assertThat(vector.getValue().getValue(Dimension.INSTANCE_ID), is(Optional.empty())); + assertThat(vector.getValue().getValue(Dimension.NODE_TYPE), is(Optional.empty())); RawFlag rawFlag = mock(RawFlag.class); when(source.fetch(eq(new FlagId("id")), any())).thenReturn(Optional.of(rawFlag)); when(rawFlag.asJsonNode()).thenReturn(BooleanNode.getTrue()); // raw flag deserializes to true - assertThat(booleanFlag.with(FetchVector.Dimension.INSTANCE_ID, "an-app").value(), equalTo(true)); + assertThat(booleanFlag.with(Dimension.INSTANCE_ID, "an-app").value(), equalTo(true)); verify(source, times(2)).fetch(any(), vector.capture()); // application was set on the (bound) flag. - assertThat(vector.getValue().getValue(FetchVector.Dimension.INSTANCE_ID), is(Optional.of("an-app"))); + assertThat(vector.getValue().getValue(Dimension.INSTANCE_ID), is(Optional.of("an-app"))); } @Test void testString() { testGeneric(Flags.defineStringFlag("string-id", "default value", List.of("owner"), "1970-01-01", "2100-01-01", "description", - "modification effect", FetchVector.Dimension.ZONE_ID, FetchVector.Dimension.HOSTNAME), + "modification effect", Dimension.ZONE_ID, Dimension.HOSTNAME), "other value"); } @@ -100,7 +100,7 @@ public class FlagsTest { instance.string = "foo"; testGeneric(Flags.defineJacksonFlag("jackson-id", defaultInstance, ExampleJacksonClass.class, - List.of("owner"), "1970-01-01", "2100-01-01", "description", "modification effect", FetchVector.Dimension.HOSTNAME), + List.of("owner"), "1970-01-01", "2100-01-01", "description", "modification effect", Dimension.HOSTNAME), instance); testGeneric(Flags.defineListFlag("jackson-list-id", List.of(defaultInstance), ExampleJacksonClass.class, List.of("owner"), "1970-01-01", "2100-01-01", "desc", "mod"), diff --git a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java index 8bf0014cbfe..4ea4f8ab638 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags.file; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.json.FlagData; @@ -55,7 +56,7 @@ public class FlagDbFileTest { // Changing value of id1, removing id2, adding id3 dataMap.remove(id2); - FlagData newData1 = new FlagData(id1, new FetchVector().with(FetchVector.Dimension.HOSTNAME, "h1")); + FlagData newData1 = new FlagData(id1, new FetchVector().with(Dimension.HOSTNAME, "h1")); dataMap.put(id1, newData1); FlagId id3 = new FlagId("id3"); FlagData data3 = new FlagData(id3, new FetchVector()); diff --git a/flags/src/test/java/com/yahoo/vespa/flags/json/ConditionTest.java b/flags/src/test/java/com/yahoo/vespa/flags/json/ConditionTest.java index a1cc2e99cbc..b283cfef0a5 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/json/ConditionTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/json/ConditionTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags.json; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.FetchVector; import org.junit.jupiter.api.Test; @@ -15,23 +16,23 @@ public class ConditionTest { @Test void testWhitelist() { String hostname1 = "host1"; - var params = new Condition.CreateParams(FetchVector.Dimension.HOSTNAME).withValues(hostname1); + var params = new Condition.CreateParams(Dimension.HOSTNAME).withValues(hostname1); Condition condition = WhitelistCondition.create(params); assertFalse(condition.test(new FetchVector())); - assertFalse(condition.test(new FetchVector().with(FetchVector.Dimension.INSTANCE_ID, "foo"))); - assertFalse(condition.test(new FetchVector().with(FetchVector.Dimension.HOSTNAME, "bar"))); - assertTrue(condition.test(new FetchVector().with(FetchVector.Dimension.HOSTNAME, hostname1))); + assertFalse(condition.test(new FetchVector().with(Dimension.INSTANCE_ID, "foo"))); + assertFalse(condition.test(new FetchVector().with(Dimension.HOSTNAME, "bar"))); + assertTrue(condition.test(new FetchVector().with(Dimension.HOSTNAME, hostname1))); } @Test void testBlacklist() { String hostname1 = "host1"; - var params = new Condition.CreateParams(FetchVector.Dimension.HOSTNAME).withValues(hostname1); + var params = new Condition.CreateParams(Dimension.HOSTNAME).withValues(hostname1); Condition condition = BlacklistCondition.create(params); assertTrue(condition.test(new FetchVector())); - assertTrue(condition.test(new FetchVector().with(FetchVector.Dimension.INSTANCE_ID, "foo"))); - assertTrue(condition.test(new FetchVector().with(FetchVector.Dimension.HOSTNAME, "bar"))); - assertFalse(condition.test(new FetchVector().with(FetchVector.Dimension.HOSTNAME, hostname1))); + assertTrue(condition.test(new FetchVector().with(Dimension.INSTANCE_ID, "foo"))); + assertTrue(condition.test(new FetchVector().with(Dimension.HOSTNAME, "bar"))); + assertFalse(condition.test(new FetchVector().with(Dimension.HOSTNAME, hostname1))); } @Test @@ -44,7 +45,7 @@ public class ConditionTest { // Test with empty fetch vector along vespa version dimension (this should never happen as the // version is always available through Vtag, although Vtag has a dummy version number for e.g. // locally run unit tests that hasn't set the release Vespa version). - var params = new Condition.CreateParams(FetchVector.Dimension.VESPA_VERSION).withPredicate(">=7.1.2"); + var params = new Condition.CreateParams(Dimension.VESPA_VERSION).withPredicate(">=7.1.2"); Condition condition = RelationalCondition.create(params); assertFalse(condition.test(new FetchVector())); } @@ -56,8 +57,8 @@ public class ConditionTest { } private boolean vespaVersionCondition(String vespaVersion, String predicate) { - var params = new Condition.CreateParams(FetchVector.Dimension.VESPA_VERSION).withPredicate(predicate); + var params = new Condition.CreateParams(Dimension.VESPA_VERSION).withPredicate(predicate); Condition condition = RelationalCondition.create(params); - return condition.test(new FetchVector().with(FetchVector.Dimension.VESPA_VERSION, vespaVersion)); + return condition.test(new FetchVector().with(Dimension.VESPA_VERSION, vespaVersion)); } } diff --git a/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java index 98c99231237..d19d262e593 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.flags.json; import com.yahoo.text.JSON; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.RawFlag; import org.junit.jupiter.api.Test; @@ -61,27 +62,27 @@ public class FlagDataTest { // First rule matches only if both conditions match verify(Optional.of("false"), vector - .with(FetchVector.Dimension.HOSTNAME, "host1") - .with(FetchVector.Dimension.INSTANCE_ID, "app2")); + .with(Dimension.HOSTNAME, "host1") + .with(Dimension.INSTANCE_ID, "app2")); verify(Optional.of("true"), vector - .with(FetchVector.Dimension.HOSTNAME, "host1") - .with(FetchVector.Dimension.INSTANCE_ID, "app3")); + .with(Dimension.HOSTNAME, "host1") + .with(Dimension.INSTANCE_ID, "app3")); // Verify unsetting a dimension with null works. verify(Optional.of("true"), vector - .with(FetchVector.Dimension.HOSTNAME, "host1") - .with(FetchVector.Dimension.INSTANCE_ID, "app3") - .with(FetchVector.Dimension.INSTANCE_ID, null)); + .with(Dimension.HOSTNAME, "host1") + .with(Dimension.INSTANCE_ID, "app3") + .with(Dimension.INSTANCE_ID, null)); // No rules apply if zone is overridden to an unknown zone - verify(Optional.empty(), vector.with(FetchVector.Dimension.ZONE_ID, "unknown zone")); + verify(Optional.empty(), vector.with(Dimension.ZONE_ID, "unknown zone")); } @Test void testPartialResolve() { FlagData data = FlagData.deserialize(json); assertEquals(data.partialResolve(vector), data); - assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.INSTANCE_ID, "app1")), + assertEquals(data.partialResolve(vector.with(Dimension.INSTANCE_ID, "app1")), FlagData.deserialize(""" { "id": "id1", @@ -102,7 +103,7 @@ public class FlagDataTest { } }""")); - assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.INSTANCE_ID, "app1")), + assertEquals(data.partialResolve(vector.with(Dimension.INSTANCE_ID, "app1")), FlagData.deserialize(""" { "id": "id1", @@ -123,7 +124,7 @@ public class FlagDataTest { } }""")); - assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.INSTANCE_ID, "app3")), + assertEquals(data.partialResolve(vector.with(Dimension.INSTANCE_ID, "app3")), FlagData.deserialize(""" { "id": "id1", @@ -154,8 +155,8 @@ public class FlagDataTest { } }""")); - assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.INSTANCE_ID, "app3") - .with(FetchVector.Dimension.HOSTNAME, "host1")), + assertEquals(data.partialResolve(vector.with(Dimension.INSTANCE_ID, "app3") + .with(Dimension.HOSTNAME, "host1")), FlagData.deserialize(""" { "id": "id1", @@ -169,8 +170,8 @@ public class FlagDataTest { } }""")); - assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.INSTANCE_ID, "app3") - .with(FetchVector.Dimension.HOSTNAME, "host3")), + assertEquals(data.partialResolve(vector.with(Dimension.INSTANCE_ID, "app3") + .with(Dimension.HOSTNAME, "host3")), FlagData.deserialize(""" { "id": "id1", @@ -191,9 +192,9 @@ public class FlagDataTest { } }""")); - assertEquals(data.partialResolve(vector.with(FetchVector.Dimension.INSTANCE_ID, "app3") - .with(FetchVector.Dimension.HOSTNAME, "host3") - .with(FetchVector.Dimension.ZONE_ID, "zone2")), + assertEquals(data.partialResolve(vector.with(Dimension.INSTANCE_ID, "app3") + .with(Dimension.HOSTNAME, "host3") + .with(Dimension.ZONE_ID, "zone2")), FlagData.deserialize(""" { "id": "id1", @@ -204,9 +205,9 @@ public class FlagDataTest { ] }""")); - FlagData fullyResolved = data.partialResolve(vector.with(FetchVector.Dimension.INSTANCE_ID, "app3") - .with(FetchVector.Dimension.HOSTNAME, "host3") - .with(FetchVector.Dimension.ZONE_ID, "zone3")); + FlagData fullyResolved = data.partialResolve(vector.with(Dimension.INSTANCE_ID, "app3") + .with(Dimension.HOSTNAME, "host3") + .with(Dimension.ZONE_ID, "zone3")); assertEquals(fullyResolved, FlagData.deserialize(""" { "id": "id1" @@ -265,7 +266,7 @@ public class FlagDataTest { }"""; FlagData data = FlagData.deserialize(json); assertTrue(JSON.equals(data.serializeToJson(), json)); - FlagData flagData = data.partialResolve(vector.with(FetchVector.Dimension.CLOUD, "gcp")); + FlagData flagData = data.partialResolve(vector.with(Dimension.CLOUD, "gcp")); assertEquals(flagData, new FlagData(data.id(), new FetchVector(), List.of())); assertTrue(flagData.isEmpty()); } diff --git a/integration/intellij/build.gradle.kts b/integration/intellij/build.gradle.kts index 0fd879d18d4..b437a1f5b28 100644 --- a/integration/intellij/build.gradle.kts +++ b/integration/intellij/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } group="ai.vespa" -version="1.4.0" // Also update pom.xml version AND the version below if this is changed +version="1.5.0" // Also update pom.xml version AND the version below if this is changed defaultTasks("buildPlugin") @@ -43,7 +43,7 @@ sourceSets { // See https://github.com/JetBrains/gradle-intellij-plugin/ intellij { - version.set("2023.1") + version.set("2023.2") } tasks { @@ -55,11 +55,11 @@ tasks { } patchPluginXml { - version.set("1.4.0") // TODO: Use one version property - sinceBuild.set("231") + version.set("1.5.0") // Keep in sync with pom.xml TODO: Use one version property + sinceBuild.set("232") // Appears on the plugin page in preferences/plugins changeNotes.set(""" - Support for IntelliJ 2023 + Support for IntelliJ 2023.2 """) } diff --git a/integration/intellij/pom.xml b/integration/intellij/pom.xml index f8fef937536..b57bd8d379f 100644 --- a/integration/intellij/pom.xml +++ b/integration/intellij/pom.xml @@ -9,7 +9,7 @@ <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-intellij</artifactId> <!-- Not used - plugin is build by gradle --> - <version>1.4.0</version> <!-- See copy-zip below, which depends on this being the same as the v. in build.gradle --> + <version>1.5.0</version> <!-- See copy-zip below, which depends on this being the same as the v. in build.gradle.kts --> <description> Maven wrapper for the gradle build of this IntelliJ plugin. </description> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/ProxyRequestHandler.java b/jdisc_core/src/main/java/com/yahoo/jdisc/ProxyRequestHandler.java index 19bb4a64a01..6edc4d105f0 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/ProxyRequestHandler.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/ProxyRequestHandler.java @@ -31,9 +31,9 @@ class ProxyRequestHandler implements DelegatedRequestHandler { @Override public ContentChannel handleRequest(Request request, ResponseHandler responseHandler) { - try (final ResourceReference requestReference = request.refer()) { + try (ResourceReference requestReference = request.refer()) { ContentChannel contentChannel; - final ResponseHandler proxyResponseHandler = new ProxyResponseHandler( + ResponseHandler proxyResponseHandler = new ProxyResponseHandler( request, new NullContentResponseHandler(responseHandler)); try { contentChannel = delegate.handleRequest(request, proxyResponseHandler); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/References.java b/jdisc_core/src/main/java/com/yahoo/jdisc/References.java index 5b845754d3d..6cae0c48c56 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/References.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/References.java @@ -7,27 +7,22 @@ package com.yahoo.jdisc; * @author bakksjo */ public class References { - // Prevents instantiation. - private References() { - } + + private References() { } /** * A {@link ResourceReference} that does nothing. * Useful for e.g. testing of resource types when reference counting is not the focus. */ - public static final ResourceReference NOOP_REFERENCE = new ResourceReference() { - @Override - public void close() { - } - }; + public static final ResourceReference NOOP_REFERENCE = () -> { }; /** * <p>Returns a {@link ResourceReference} that invokes {@link SharedResource#release()} on * {@link ResourceReference#close() close}. Useful for treating the "main" reference of a {@link SharedResource} * just as any other reference obtained by calling {@link SharedResource#refer()}. Example:</p> * <pre> - * final Request request = new Request(...); - * try (final ResourceReference ref = References.fromResource(request)) { + * Request request = new Request(...); + * try (ResourceReference ref = References.fromResource(request)) { * .... * } * // The request will be released on exit from the try block. @@ -36,12 +31,8 @@ public class References { * @param resource The resource to create a ResourceReference for. * @return a ResourceReference whose close() method will call release() on the given resource. */ - public static ResourceReference fromResource(final SharedResource resource) { - return new ResourceReference() { - @Override - public void close() { - resource.release(); - } - }; + public static ResourceReference fromResource(SharedResource resource) { + return resource::release; } + } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java index 61f7c04ec9a..91741fc7d4d 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java @@ -41,7 +41,7 @@ public class Request extends AbstractResource { private final HeaderFields headers = new HeaderFields(); private final Container container; private final Request parent; - private final ResourceReference parentReference; + private final ResourceReference resourceReference; private final long creationTime; private final boolean serverRequest; private final URI uri; @@ -83,15 +83,17 @@ public class Request extends AbstractResource { this(current, uri, true); } - public Request(CurrentContainer current, URI uri, boolean isServerRequest) { this(current, uri, isServerRequest, -1); } + public Request(CurrentContainer current, URI uri, boolean isServerRequest) { + this(current, uri, isServerRequest, -1); + } public Request(CurrentContainer current, URI uri, boolean isServerRequest, long creationTime) { - parent = null; - parentReference = null; - serverRequest = isServerRequest; + this.parent = null; + this.container = current.newReference(uri, this); + this.resourceReference = container::release; this.uri = uri.normalize(); - container = current.newReference(uri, this); this.creationTime = creationTime >= 0 ? creationTime : container.currentTimeMillis(); + this.serverRequest = isServerRequest; } @@ -121,11 +123,11 @@ public class Request extends AbstractResource { */ public Request(Request parent, URI uri) { this.parent = parent; - container = null; - creationTime = parent.container().currentTimeMillis(); - serverRequest = false; + this.container = null; + this.resourceReference = parent.refer(this); this.uri = uri.normalize(); - parentReference = this.parent.refer(this); + this.creationTime = parent.container().currentTimeMillis(); + this.serverRequest = false; } /** Returns the {@link Container} for which this Request was created */ @@ -140,7 +142,7 @@ public class Request extends AbstractResource { public URI getUri() { return uri; } /** - * Returns whether or not this Request was created by a {@link ServerProvider}. The value of this is used by + * Returns whether this Request was created by a {@link ServerProvider}. The value of this is used by * {@link Container#resolveHandler(Request)} to decide whether to match against server- or client-bindings. * * @return true, if this is a server request @@ -395,12 +397,7 @@ public class Request extends AbstractResource { @Override protected void destroy() { - if (parentReference != null) { - parentReference.close(); - } - if (container != null) { - container.release(); - } + resourceReference.close(); } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/ResourceReference.java b/jdisc_core/src/main/java/com/yahoo/jdisc/ResourceReference.java index 6209c74374e..449c597bdc7 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/ResourceReference.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/ResourceReference.java @@ -6,8 +6,8 @@ package com.yahoo.jdisc; * * <p>Implements {@link AutoCloseable} so that it can be used in try-with-resources statements. Example</p> * <pre> - * void doSomethingWithRequest(final Request request) { - * try (final ResourceReference ref = request.refer()) { + * void doSomethingWithRequest(Request request) { + * try (ResourceReference ref = request.refer()) { * // Do something with request * } * // ref.close() will be called automatically on exit from the try block, releasing the reference on 'request'. diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerBuilder.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerBuilder.java index e172b5d1f64..6550d9b5386 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerBuilder.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerBuilder.java @@ -13,6 +13,9 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.toUnmodifiableMap; /** * <p>This is the inactive, mutable {@link Container}. Because it requires references to the application internals, it @@ -35,9 +38,7 @@ public class ContainerBuilder { public ContainerBuilder(Iterable<Module> guiceModules) { this.guiceModules.installAll(guiceModules); this.guiceModules.install(new AbstractModule() { - - @Override - public void configure() { + @Override public void configure() { bind(ContainerBuilder.class).toInstance(ContainerBuilder.this); } }); @@ -73,42 +74,26 @@ public class ContainerBuilder { return serverBindings.get(BindingSet.DEFAULT); } - public BindingRepository<RequestHandler> serverBindings(String setName) { - BindingRepository<RequestHandler> ret = serverBindings.get(setName); - if (ret == null) { - ret = new BindingRepository<>(); - serverBindings.put(setName, ret); - } - return ret; + public BindingRepository<RequestHandler> clientBindings() { + return clientBindings.get(BindingSet.DEFAULT); } - public Map<String, BindingSet<RequestHandler>> activateServerBindings() { - Map<String, BindingSet<RequestHandler>> ret = new HashMap<>(); - for (Map.Entry<String, BindingRepository<RequestHandler>> entry : serverBindings.entrySet()) { - ret.put(entry.getKey(), entry.getValue().activate()); - } - return ImmutableMap.copyOf(ret); + public BindingRepository<RequestHandler> serverBindings(String setName) { + return serverBindings.computeIfAbsent(setName, __ -> new BindingRepository<>()); } - public BindingRepository<RequestHandler> clientBindings() { - return clientBindings.get(BindingSet.DEFAULT); + public BindingRepository<RequestHandler> clientBindings(String setName) { + return clientBindings.computeIfAbsent(setName, __ -> new BindingRepository<>()); } - public BindingRepository<RequestHandler> clientBindings(String setName) { - BindingRepository<RequestHandler> ret = clientBindings.get(setName); - if (ret == null) { - ret = new BindingRepository<>(); - clientBindings.put(setName, ret); - } - return ret; + public Map<String, BindingSet<RequestHandler>> activateServerBindings() { + return serverBindings.entrySet().stream().collect(toUnmodifiableMap(entry -> entry.getKey(), + entry -> entry.getValue().activate())); } public Map<String, BindingSet<RequestHandler>> activateClientBindings() { - Map<String, BindingSet<RequestHandler>> ret = new HashMap<>(); - for (Map.Entry<String, BindingRepository<RequestHandler>> entry : clientBindings.entrySet()) { - ret.put(entry.getKey(), entry.getValue().activate()); - } - return ImmutableMap.copyOf(ret); + return clientBindings.entrySet().stream().collect(toUnmodifiableMap(entry -> entry.getKey(), + entry -> entry.getValue().activate())); } @SuppressWarnings({ "unchecked" }) diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java index 53fb81fb6db..e2d2da660c7 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java @@ -37,12 +37,12 @@ public class ActiveContainer extends AbstractResource implements CurrentContaine serverProviders = builder.serverProviders().activate(); serverProviders.forEach(resourceReferences::retain); serverBindings = builder.activateServerBindings(); - serverBindings.forEach( - (ignoredName, bindingSet) -> bindingSet.forEach( + serverBindings.values().forEach( + bindingSet -> bindingSet.forEach( binding -> resourceReferences.retain(binding.getValue()))); clientBindings = builder.activateClientBindings(); - clientBindings.forEach( - (ignoredName, bindingSet) -> bindingSet.forEach( + clientBindings.values().forEach( + bindingSet -> bindingSet.forEach( binding -> resourceReferences.retain(binding.getValue()))); bindingSetSelector = builder.getInstance(BindingSetSelector.class); timeoutMgr = builder.getInstance(TimeoutManagerImpl.class); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureResponse.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureResponse.java index d5eda23a8d6..4730535640c 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureResponse.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureResponse.java @@ -30,18 +30,10 @@ public final class FutureResponse extends CompletableFuture<Response> implements * * @param content The content channel for the Response. */ - public FutureResponse(final ContentChannel content) { - this(new ResponseHandler() { - - @Override - public ContentChannel handleResponse(Response response) { - return content; - } - }); + public FutureResponse(ContentChannel content) { + this(response -> content); } - public void addListener(Runnable r, Executor e) { whenCompleteAsync((__, ___) -> r.run(), e); } - /** * <p>Constructs a new FutureResponse that calls the given {@link ResponseHandler} when {@link * #handleResponse(Response)} is invoked.</p> @@ -52,6 +44,8 @@ public final class FutureResponse extends CompletableFuture<Response> implements this.handler = handler; } + public void addListener(Runnable r, Executor e) { whenCompleteAsync((__, ___) -> r.run(), e); } + @Override public ContentChannel handleResponse(Response response) { complete(response); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java index b60b62e3f86..ef420d69f16 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java @@ -27,7 +27,7 @@ import java.util.concurrent.TimeoutException; * * <p>The following is a simple example on how to use this class:</p> * <pre> - * public void handleRequest(final Request parent, final ResponseHandler handler) { + * public void handleRequest(Request parent, ResponseHandler handler) { * new RequestDispatch() { * @Override * protected Request newRequest() { @@ -80,8 +80,8 @@ public abstract class RequestDispatch implements Future<Response>, ResponseHandl * @return The ContentChannel to write the Request's content to. */ public final ContentChannel connect() { - final Request request = newRequest(); - try (final ResourceReference ref = References.fromResource(request)) { + Request request = newRequest(); + try (ResourceReference ref = References.fromResource(request)) { return request.connect(futureResponse); } } diff --git a/jrt/src/com/yahoo/jrt/TransportThread.java b/jrt/src/com/yahoo/jrt/TransportThread.java index 68b7c23b36b..870d1ce7adc 100644 --- a/jrt/src/com/yahoo/jrt/TransportThread.java +++ b/jrt/src/com/yahoo/jrt/TransportThread.java @@ -296,6 +296,9 @@ public class TransportThread { */ public TransportThread sync() { SyncCmd cmd = new SyncCmd(); + if (Thread.currentThread() == thread) { + log.log(Level.WARNING, "Attempting to sync " + thread + " with itself, which will deadlock"); + } if (postCommand(cmd)) { cmd.waitDone(); } else { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java b/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java index 7e8286d8793..805242665ca 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java @@ -63,13 +63,11 @@ public final class IntermediateSession implements MessageHandler, ReplyHandler, * @param routable the routable to forward. */ public void forward(Routable routable) { - if (routable instanceof Reply) { - Reply reply = (Reply)routable; - ReplyHandler handler = reply.popHandler(); - handler.handleReply(reply); + if (routable instanceof Reply reply) { + reply.popHandler().handleReply(reply); } else { routable.pushHandler(this); - mbus.handleMessage((Message)routable); + mbus.handleMessage((Message) routable); } } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index eeea999cc14..4a443a9fde5 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -367,7 +367,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * sessions from receiving data from message bus.</p> * * @param name The name of the session to remove. - * @param broadcastName Whether or not session name was broadcast. + * @param broadcastName Whether session name was broadcast. */ public synchronized void unregisterSession(String name, boolean broadcastName) { net.unregisterSession(name, this, broadcastName); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java index 794ac152c9f..974eb54d779 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java @@ -42,7 +42,7 @@ public class Messenger implements Runnable { * * @param task The task to add. */ - void addRecurrentTask(final Task task) { + void addRecurrentTask(Task task) { children.add(task); } @@ -64,7 +64,7 @@ public class Messenger implements Runnable { * @param msg The message to send. * @param handler The handler to send to. */ - public void deliverMessage(final Message msg, final MessageHandler handler) { + public void deliverMessage(Message msg, MessageHandler handler) { if (destroyed.get()) { msg.discard(); } else { @@ -80,7 +80,7 @@ public class Messenger implements Runnable { * @param reply The reply to return. * @param handler The handler to return to. */ - public void deliverReply(final Reply reply, final ReplyHandler handler) { + public void deliverReply(Reply reply, ReplyHandler handler) { if (destroyed.get()) { reply.discard(); } else { @@ -95,7 +95,7 @@ public class Messenger implements Runnable { * * @param task The task to enqueue. */ - public void enqueue(final Task task) { + public void enqueue(Task task) { if (destroyed.get()) { task.destroy(); return; @@ -116,7 +116,7 @@ public class Messenger implements Runnable { if (Thread.currentThread() == thread) { return; // no need to wait for self } - final SyncTask task = new SyncTask(); + SyncTask task = new SyncTask(); enqueue(task); task.await(); } @@ -140,7 +140,7 @@ public class Messenger implements Runnable { } } thread.join(); - } catch (final InterruptedException e) { + } catch (InterruptedException e) { // ignore } done = true; @@ -161,7 +161,7 @@ public class Messenger implements Runnable { } else { wait(timeoutMS); } - } catch (final InterruptedException e) { + } catch (InterruptedException e) { continue; } } @@ -175,26 +175,26 @@ public class Messenger implements Runnable { if (task != null) { try { task.run(); - } catch (final Exception e) { + } catch (Exception e) { log.log(Level.SEVERE, "An exception was thrown while running " + task.getClass().getName(), e); } try { task.destroy(); - } catch (final Exception e) { + } catch (Exception e) { log.warning("An exception was thrown while destroying " + task.getClass().getName() + ": " + e); log.warning("Someone, somewhere might have to wait indefinitely for something."); } } - for (final Task child : children) { + for (Task child : children) { child.run(); } } - for (final Task child : children) { + for (Task child : children) { child.destroy(); } synchronized (this) { while (!queue.isEmpty()) { - final Task task = queue.poll(); + Task task = queue.poll(); task.destroy(); } notify(); @@ -236,7 +236,7 @@ public class Messenger implements Runnable { public void await() { try { latch.await(); - } catch (final InterruptedException e) { + } catch (InterruptedException e) { // ignore } } diff --git a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java index 83fefbabba1..05d51967166 100644 --- a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java @@ -11,6 +11,7 @@ public enum ControllerMetrics implements VespaMetrics { DEPLOYMENT_JOBS_QUEUED("deployment.jobsQueued", Unit.TASK, "The number of deployment jobs queued"), DEPLOYMENT_JOBS_ACTIVE("deployment.jobsActive", Unit.TASK, "The number of deployment jobs active"), + DEPLOYMENT_EXECUTOR_SIZE("deployment.executorSize", Unit.THREAD, "The number of deployment jobs that can run in parallel"), DEPLOYMENT_START("deployment.start", Unit.DEPLOYMENT, "The number of started deployment jobs"), DEPLOYMENT_NODE_ALLOCATION_FAILURE("deployment.nodeAllocationFailure", Unit.DEPLOYMENT, "The number of deployments failed due to node allocation failures"), DEPLOYMENT_ENDPOINT_CERTIFICATE_TIMEOUT("deployment.endpointCertificateTimeout", Unit.DEPLOYMENT, "The number of deployments failed due to timeout acquiring endpoint certificate"), diff --git a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java index 2d8279db97e..bca52c03892 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java @@ -146,6 +146,7 @@ public class InfrastructureMetricSet { addMetric(metrics, ControllerMetrics.DEPLOYMENT_JOBS_QUEUED, EnumSet.of(count, sum)); addMetric(metrics, ControllerMetrics.DEPLOYMENT_JOBS_ACTIVE, EnumSet.of(count, sum)); + addMetric(metrics, ControllerMetrics.DEPLOYMENT_EXECUTOR_SIZE, EnumSet.of(max)); addMetric(metrics, ControllerMetrics.DEPLOYMENT_ABORT.count()); addMetric(metrics, ControllerMetrics.DEPLOYMENT_DURATION.max()); addMetric(metrics, ControllerMetrics.DEPLOYMENT_AVERAGE_DURATION.max()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 620026c4ac6..b29bfd9af62 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -59,7 +59,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { int failures = 0; outer: for (var applicationNodes : activeNodesByApplication().entrySet()) { - boolean enabled = enabledFlag.with(FetchVector.Dimension.INSTANCE_ID, + boolean enabled = enabledFlag.with(Dimension.INSTANCE_ID, applicationNodes.getKey().serializedForm()).value(); if (!enabled) continue; for (var clusterNodes : nodesByCluster(applicationNodes.getValue()).entrySet()) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java index 3cec8483a45..bdff94e011d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java @@ -5,7 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; @@ -66,7 +66,7 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer { private boolean shouldMaintain(ApplicationId id) { BooleanFlag skipMaintenanceDeployment = PermanentFlags.SKIP_MAINTENANCE_DEPLOYMENT.bindTo(flagSource) - .with(FetchVector.Dimension.INSTANCE_ID, id.serializedForm()); + .with(Dimension.INSTANCE_ID, id.serializedForm()); return ! skipMaintenanceDeployment.value(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 1e9adea4e95..b2db5977109 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -19,7 +19,7 @@ import java.util.Map; import java.util.TreeMap; import static com.yahoo.config.provision.NodeResources.Architecture; -import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID; +import static com.yahoo.vespa.flags.Dimension.INSTANCE_ID; import static java.util.Objects.requireNonNull; /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index e6f2dc0fbfe..239b962360b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.ZoneEndpoint; @@ -15,7 +14,7 @@ import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; @@ -45,7 +44,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import static com.yahoo.vespa.applicationmodel.TenantId.HOSTED_VESPA; import static com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec.preProvisionOwner; import static com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec.preProvisionSpec; import static java.util.stream.Collectors.groupingBy; @@ -369,7 +367,7 @@ public class LoadBalancerProvisioner { LoadBalancer currentLoadBalancer, ZoneEndpoint zoneEndpoint, CloudAccount cloudAccount) { - boolean shouldDeactivateRouting = deactivateRouting.with(FetchVector.Dimension.INSTANCE_ID, + boolean shouldDeactivateRouting = deactivateRouting.with(Dimension.INSTANCE_ID, id.application().serializedForm()) .value(); Set<Real> reals = shouldDeactivateRouting ? Set.of() : realsOf(nodes, cloudAccount); @@ -427,7 +425,7 @@ public class LoadBalancerProvisioner { /** Find IP addresses reachable by the load balancer service */ private Set<String> reachableIpAddresses(Node node, CloudAccount cloudAccount) { Set<String> reachable = new LinkedHashSet<>(node.ipConfig().primary()); - boolean forceIpv6 = ipv6AwsTargetGroups.with(FetchVector.Dimension.CLOUD_ACCOUNT, cloudAccount.account()).value(); + boolean forceIpv6 = ipv6AwsTargetGroups.with(Dimension.CLOUD_ACCOUNT, cloudAccount.account()).value(); var protocol = forceIpv6 ? LoadBalancerService.Protocol.ipv6 : service.protocol(node.cloudAccount().isExclave(nodeRepository.zone())); // Remove addresses unreachable by the load balancer service diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index dcf4e3160d4..8c52f389daf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -9,7 +9,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.net.HostName; -import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -94,9 +94,9 @@ class NodeAllocation { this.nextIndex = nextIndex; this.nodeRepository = nodeRepository; this.requiredHostFlavor = Optional.of(PermanentFlags.HOST_FLAVOR.bindTo(nodeRepository.flagSource()) - .with(FetchVector.Dimension.INSTANCE_ID, application.serializedForm()) - .with(FetchVector.Dimension.CLUSTER_TYPE, cluster.type().name()) - .with(FetchVector.Dimension.CLUSTER_ID, cluster.id().value()) + .with(Dimension.INSTANCE_ID, application.serializedForm()) + .with(Dimension.CLUSTER_TYPE, cluster.type().name()) + .with(Dimension.CLUSTER_ID, cluster.id().value()) .value()) .filter(s -> !s.isBlank()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index dd6b0b5a6df..5c379fb1608 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -13,7 +13,7 @@ import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Dimension; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.provision.Node; @@ -199,13 +199,13 @@ class NodesResponse extends SlimeJsonResponse { private Version resolveVersionFlag(StringFlag flag, Node node, Allocation allocation) { String value = flag - .with(FetchVector.Dimension.HOSTNAME, node.hostname()) - .with(FetchVector.Dimension.NODE_TYPE, node.type().name()) - .with(FetchVector.Dimension.TENANT_ID, allocation.owner().tenant().value()) - .with(FetchVector.Dimension.INSTANCE_ID, allocation.owner().serializedForm()) - .with(FetchVector.Dimension.CLUSTER_TYPE, allocation.membership().cluster().type().name()) - .with(FetchVector.Dimension.CLUSTER_ID, allocation.membership().cluster().id().value()) - .with(FetchVector.Dimension.VESPA_VERSION, allocation.membership().cluster().vespaVersion().toFullString()) + .with(Dimension.HOSTNAME, node.hostname()) + .with(Dimension.NODE_TYPE, node.type().name()) + .with(Dimension.TENANT_ID, allocation.owner().tenant().value()) + .with(Dimension.INSTANCE_ID, allocation.owner().serializedForm()) + .with(Dimension.CLUSTER_TYPE, allocation.membership().cluster().type().name()) + .with(Dimension.CLUSTER_ID, allocation.membership().cluster().id().value()) + .with(Dimension.VESPA_VERSION, allocation.membership().cluster().vespaVersion().toFullString()) .value(); return value.isEmpty() ? diff --git a/parent/pom.xml b/parent/pom.xml index d4a2036eb5f..7642b87ac7b 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -317,7 +317,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.14.0</version> + <version>5.14.1</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> |