diff options
| author | Ola Aunronning <olaa@yahooinc.com> | 2023-04-26 09:36:43 +0200 |
|---|---|---|
| committer | Ola Aunronning <olaa@yahooinc.com> | 2023-04-26 09:36:43 +0200 |
| commit | 48b7ffe757ffa25bc3ca1eeaab8153db30623fa3 (patch) | |
| tree | b41007ada3c803cc3282cc0556acbb13f3642e6a | |
| parent | d46b56f3f8577d8c4cc4b08f5f13b3b983ef7d2a (diff) | |
| parent | 9a4376dae10e986c7061633e5a02f18c24a951da (diff) | |
Merge branch 'master' into olaa/versioned-feature-flag
264 files changed, 2923 insertions, 3099 deletions
diff --git a/client/go/go.mod b/client/go/go.mod index 18e3853868d..94f69c8286a 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -13,8 +13,8 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 github.com/zalando/go-keyring v0.1.1 - golang.org/x/net v0.8.0 - golang.org/x/sys v0.6.0 + golang.org/x/net v0.9.0 + golang.org/x/sys v0.7.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) @@ -28,7 +28,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/stretchr/objx v0.1.1 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/client/go/go.sum b/client/go/go.sum index 2af8bb1e4c0..ac662c9fd43 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -47,16 +47,16 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/zalando/go-keyring v0.1.1 h1:w2V9lcx/Uj4l+dzAf1m9s+DJ1O8ROkEHnynonHjTcYE= github.com/zalando/go-keyring v0.1.1/go.mod h1:OIC+OZ28XbmwFxU/Rp9V7eKzZjamBJwRzC8UFJH9+L8= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/client/go/internal/admin/prog/common_env.go b/client/go/internal/admin/prog/common_env.go deleted file mode 100644 index f743716a64e..00000000000 --- a/client/go/internal/admin/prog/common_env.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package prog - -import ( - "os" - "strings" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func (spec *Spec) configureCommonEnv() { - os.Unsetenv(envvars.LD_PRELOAD) - spec.Setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") - spec.Setenv(envvars.GLIBCXX_FORCE_NEW, "1") - spec.Setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") - spec.Setenv(envvars.MALLOC_ARENA_MAX, "1") - - // fallback from old env.vars: - spec.considerEnvFallback(envvars.VESPA_USE_HUGEPAGES_LIST, envvars.HUGEPAGES_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_MADVISE_LIST, envvars.MADVISE_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC, envvars.VESPAMALLOC_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_D, envvars.VESPAMALLOCD_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_DST, envvars.VESPAMALLOCDST_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_NO_VESPAMALLOC, envvars.NO_VESPAMALLOC_LIST) - // other fallbacks: - spec.considerFallback(envvars.ROOT, vespa.FindHome()) - spec.considerFallback(envvars.VESPA_USER, vespa.FindVespaUser()) - spec.considerFallback(envvars.VESPA_USE_HUGEPAGES_LIST, "all") - spec.considerFallback(envvars.VESPA_USE_VESPAMALLOC, "all") - spec.considerFallback(envvars.VESPA_USE_NO_VESPAMALLOC, strings.Join([]string{ - "vespa-rpc-invoke", - "vespa-get-config", - "vespa-sentinel-cmd", - "vespa-route", - "vespa-proton-cmd", - "vespa-configproxy-cmd", - "vespa-config-status", - }, " ")) - -} diff --git a/client/go/internal/admin/prog/hugepages.go b/client/go/internal/admin/prog/hugepages.go index c6f019937ff..b66d512d4c9 100644 --- a/client/go/internal/admin/prog/hugepages.go +++ b/client/go/internal/admin/prog/hugepages.go @@ -9,7 +9,7 @@ import ( ) func (spec *Spec) ConfigureHugePages() { - if spec.matchesListEnv(envvars.VESPA_USE_HUGEPAGES_LIST) { + if spec.MatchesListEnv(envvars.VESPA_USE_HUGEPAGES_LIST) { trace.Debug("setting", envvars.VESPA_USE_HUGEPAGES, "= 'yes'") spec.Setenv(envvars.VESPA_USE_HUGEPAGES, "yes") } diff --git a/client/go/internal/admin/prog/madvise.go b/client/go/internal/admin/prog/madvise.go index 48986a12182..967823d956b 100644 --- a/client/go/internal/admin/prog/madvise.go +++ b/client/go/internal/admin/prog/madvise.go @@ -9,7 +9,7 @@ import ( ) func (spec *Spec) ConfigureUseMadvise() { - limit := spec.valueFromListEnv(envvars.VESPA_USE_MADVISE_LIST) + limit := spec.ValueFromListEnv(envvars.VESPA_USE_MADVISE_LIST) if limit != "" { trace.Trace("shall use madvise with limit", limit, "as set in", envvars.VESPA_USE_MADVISE_LIST) spec.Setenv(envvars.VESPA_MALLOC_MADVISE_LIMIT, limit) diff --git a/client/go/internal/admin/prog/spec_env.go b/client/go/internal/admin/prog/spec_env.go index 4fa40695acb..c88ec963812 100644 --- a/client/go/internal/admin/prog/spec_env.go +++ b/client/go/internal/admin/prog/spec_env.go @@ -50,17 +50,17 @@ func (spec *Spec) EffectiveEnv() []string { return envVec } -func (spec *Spec) considerFallback(varName, varValue string) { +func (spec *Spec) ConsiderFallback(varName, varValue string) { if spec.Getenv(varName) == "" && varValue != "" { spec.Setenv(varName, varValue) } } -func (spec *Spec) considerEnvFallback(targetVar, fallbackVar string) { - spec.considerFallback(targetVar, spec.Getenv(fallbackVar)) +func (spec *Spec) ConsiderEnvFallback(targetVar, fallbackVar string) { + spec.ConsiderFallback(targetVar, spec.Getenv(fallbackVar)) } -func (p *Spec) matchesListEnv(envVarName string) bool { +func (p *Spec) MatchesListEnv(envVarName string) bool { return p.matchesListString(p.Getenv(envVarName)) } @@ -80,7 +80,7 @@ func (p *Spec) matchesListString(env string) bool { return false } -func (p *Spec) valueFromListEnv(envVarName string) string { +func (p *Spec) ValueFromListEnv(envVarName string) string { return p.valueFromListString(p.Getenv(envVarName)) } diff --git a/client/go/internal/admin/prog/valgrind.go b/client/go/internal/admin/prog/valgrind.go index 2d7f0a597d9..b949102d6bd 100644 --- a/client/go/internal/admin/prog/valgrind.go +++ b/client/go/internal/admin/prog/valgrind.go @@ -21,25 +21,18 @@ const ( func (p *Spec) ConfigureValgrind() { p.shouldUseValgrind = false p.shouldUseCallgrind = false - env := p.Getenv(envvars.VESPA_USE_VALGRIND) - allValgrind := env == "all" - parts := strings.Split(env, " ") - for _, part := range parts { - if p.BaseName == part || allValgrind { - trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env) - backticks := util.BackTicksWithStderr - out, err := backticks.Run(VALGRIND_PROG, "--help") - if err != nil { - trace.Trace("trial run of valgrind fails:", err, "=>", out) - return - } - if opts := p.Getenv(envvars.VESPA_VALGRIND_OPT); strings.Contains(opts, "callgrind") { - p.shouldUseCallgrind = true - } - p.shouldUseValgrind = true + if p.MatchesListEnv(envvars.VESPA_USE_VALGRIND) { + trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND) + backticks := util.BackTicksWithStderr + out, err := backticks.Run(VALGRIND_PROG, "--help") + if err != nil { + trace.Trace("trial run of valgrind fails:", err, "=>", out) return } - trace.Debug("checking", envvars.VESPA_USE_VALGRIND, ":", p.BaseName, "!=", part) + if opts := p.Getenv(envvars.VESPA_VALGRIND_OPT); strings.Contains(opts, "callgrind") { + p.shouldUseCallgrind = true + } + p.shouldUseValgrind = true } } diff --git a/client/go/internal/admin/prog/vespamalloc.go b/client/go/internal/admin/prog/vespamalloc.go index 439935770d7..e66c9e5d966 100644 --- a/client/go/internal/admin/prog/vespamalloc.go +++ b/client/go/internal/admin/prog/vespamalloc.go @@ -27,7 +27,7 @@ func vespaMallocLib(suf string) string { func (p *Spec) ConfigureVespaMalloc() { p.shouldUseVespaMalloc = false - if p.matchesListEnv(envvars.VESPA_USE_NO_VESPAMALLOC) { + if p.MatchesListEnv(envvars.VESPA_USE_NO_VESPAMALLOC) { trace.Trace("use no vespamalloc:", p.BaseName) return } @@ -36,11 +36,11 @@ func (p *Spec) ConfigureVespaMalloc() { return } var useFile string - if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_DST) { + if p.MatchesListEnv(envvars.VESPA_USE_VESPAMALLOC_DST) { useFile = vespaMallocLib("libvespamallocdst16.so") - } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_D) { + } else if p.MatchesListEnv(envvars.VESPA_USE_VESPAMALLOC_D) { useFile = vespaMallocLib("libvespamallocd.so") - } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC) { + } else if p.MatchesListEnv(envvars.VESPA_USE_VESPAMALLOC) { useFile = vespaMallocLib("libvespamalloc.so") } trace.Trace("use file:", useFile) @@ -51,7 +51,7 @@ func (p *Spec) ConfigureVespaMalloc() { otherFile := vespaMallocLib("libvespa_load_as_huge.so") useFile = fmt.Sprintf("%s:%s", useFile, otherFile) } - p.considerEnvFallback(envvars.VESPA_MALLOC_HUGEPAGES, envvars.VESPA_USE_HUGEPAGES) + p.ConsiderEnvFallback(envvars.VESPA_MALLOC_HUGEPAGES, envvars.VESPA_USE_HUGEPAGES) p.vespaMallocPreload = useFile p.shouldUseVespaMalloc = true } diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/common_env.go b/client/go/internal/admin/vespa-wrapper/startcbinary/common_env.go index 6bc730b5119..07ec19bf7e5 100644 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/common_env.go +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/common_env.go @@ -8,40 +8,30 @@ import ( "strings" "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/admin/prog" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) -func (spec *ProgSpec) considerFallback(varName, varValue string) { - if spec.getenv(varName) == "" && varValue != "" { - spec.setenv(varName, varValue) - } -} - -func (spec *ProgSpec) considerEnvFallback(targetVar, fallbackVar string) { - spec.considerFallback(targetVar, spec.getenv(fallbackVar)) -} - -func (spec *ProgSpec) configureCommonEnv() { +func configureCommonEnv(spec *prog.Spec) { os.Unsetenv(envvars.LD_PRELOAD) - spec.setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") - spec.setenv(envvars.GLIBCXX_FORCE_NEW, "1") - spec.setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") - spec.setenv(envvars.MALLOC_ARENA_MAX, "1") + spec.Setenv(envvars.STD_THREAD_PREVENT_TRY_CATCH, "true") + spec.Setenv(envvars.GLIBCXX_FORCE_NEW, "1") + spec.Setenv(envvars.LD_LIBRARY_PATH, vespa.FindHome()+"/lib64:/opt/vespa-deps/lib64") + spec.Setenv(envvars.MALLOC_ARENA_MAX, "1") // fallback from old env.vars: - spec.considerEnvFallback(envvars.VESPA_USE_HUGEPAGES_LIST, envvars.HUGEPAGES_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_MADVISE_LIST, envvars.MADVISE_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC, envvars.VESPAMALLOC_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_D, envvars.VESPAMALLOCD_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_VESPAMALLOC_DST, envvars.VESPAMALLOCDST_LIST) - spec.considerEnvFallback(envvars.VESPA_USE_NO_VESPAMALLOC, envvars.NO_VESPAMALLOC_LIST) + spec.ConsiderEnvFallback(envvars.VESPA_USE_HUGEPAGES_LIST, envvars.HUGEPAGES_LIST) + spec.ConsiderEnvFallback(envvars.VESPA_USE_MADVISE_LIST, envvars.MADVISE_LIST) + spec.ConsiderEnvFallback(envvars.VESPA_USE_VESPAMALLOC, envvars.VESPAMALLOC_LIST) + spec.ConsiderEnvFallback(envvars.VESPA_USE_VESPAMALLOC_D, envvars.VESPAMALLOCD_LIST) + spec.ConsiderEnvFallback(envvars.VESPA_USE_VESPAMALLOC_DST, envvars.VESPAMALLOCDST_LIST) + spec.ConsiderEnvFallback(envvars.VESPA_USE_NO_VESPAMALLOC, envvars.NO_VESPAMALLOC_LIST) // other fallbacks: - spec.considerFallback(envvars.ROOT, vespa.FindHome()) - spec.considerFallback(envvars.VESPA_USER, vespa.FindVespaUser()) - spec.considerFallback(envvars.VESPA_USE_HUGEPAGES_LIST, "all") - spec.considerFallback(envvars.VESPA_USE_VESPAMALLOC, "all") - spec.considerFallback(envvars.VESPA_USE_NO_VESPAMALLOC, strings.Join([]string{ + spec.ConsiderFallback(envvars.ROOT, vespa.FindHome()) + spec.ConsiderFallback(envvars.VESPA_USER, vespa.FindVespaUser()) + spec.ConsiderFallback(envvars.VESPA_USE_HUGEPAGES_LIST, "all") + spec.ConsiderFallback(envvars.VESPA_USE_VESPAMALLOC, "all") + spec.ConsiderFallback(envvars.VESPA_USE_NO_VESPAMALLOC, strings.Join([]string{ "vespa-rpc-invoke", "vespa-get-config", "vespa-sentinel-cmd", @@ -53,31 +43,16 @@ func (spec *ProgSpec) configureCommonEnv() { } -func (spec *ProgSpec) configureHugePages() { - if spec.matchesListEnv(envvars.VESPA_USE_HUGEPAGES_LIST) { - spec.setenv(envvars.VESPA_USE_HUGEPAGES, "yes") - } -} - -func (spec *ProgSpec) configureUseMadvise() { - limit := spec.valueFromListEnv(envvars.VESPA_USE_MADVISE_LIST) - if limit != "" { - trace.Trace("shall use madvise with limit", limit, "as set in", envvars.VESPA_USE_MADVISE_LIST) - spec.setenv(envvars.VESPA_MALLOC_MADVISE_LIMIT, limit) - return - } -} - -func (spec *ProgSpec) configurePath() { +func configurePath(spec *prog.Spec) { // Prefer newer gdb and pstack: - spec.prependPath("/opt/rh/gcc-toolset-12/root/usr/bin") + prependPath("/opt/rh/gcc-toolset-12/root/usr/bin", spec) // Maven is needed for tester applications: - spec.prependPath(vespa.FindHome() + "/local/maven/bin") - spec.prependPath(vespa.FindHome() + "/bin64") - spec.prependPath(vespa.FindHome() + "/bin") + prependPath(vespa.FindHome()+"/local/maven/bin", spec) + prependPath(vespa.FindHome()+"/bin64", spec) + prependPath(vespa.FindHome()+"/bin", spec) // how to find the "java" program? // should be available in $VESPA_HOME/bin or JAVA_HOME - if javaHome := spec.getenv(envvars.JAVA_HOME); javaHome != "" { - spec.prependPath(javaHome + "/bin") + if javaHome := spec.Getenv(envvars.JAVA_HOME); javaHome != "" { + prependPath(javaHome+"/bin", spec) } } diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/numactl.go b/client/go/internal/admin/vespa-wrapper/startcbinary/numactl.go deleted file mode 100644 index fe091dedba9..00000000000 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/numactl.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package startcbinary - -import ( - "fmt" - "strconv" - "strings" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -func (p *ProgSpec) configureNumaCtl() { - p.shouldUseNumaCtl = false - p.numaSocket = -1 - if p.getenv(envvars.VESPA_NO_NUMACTL) != "" { - return - } - backticks := util.BackTicksIgnoreStderr - out, err := backticks.Run("numactl", "--hardware") - trace.Debug("numactl --hardware says:", out) - if err != nil { - trace.Trace("numactl error:", err) - return - } - outfoo, errfoo := backticks.Run("numactl", "--interleave", "all", "echo", "foo") - if errfoo != nil { - trace.Trace("cannot run with numactl:", errfoo) - return - } - if outfoo != "foo\n" { - trace.Trace("bad numactl output:", outfoo) - return - } - p.shouldUseNumaCtl = true - if affinity := p.getenv(envvars.VESPA_AFFINITY_CPU_SOCKET); affinity != "" { - wantSocket, _ := strconv.Atoi(affinity) - trace.Debug("want socket:", wantSocket) - parts := strings.Fields(out) - for idx := 0; idx+2 < len(parts); idx++ { - if parts[idx] == "available:" && parts[idx+2] == "nodes" { - numSockets, _ := strconv.Atoi(parts[idx+1]) - trace.Debug("numSockets:", numSockets) - if numSockets > 1 { - p.numaSocket = (wantSocket % numSockets) - return - } - } - } - } -} - -func (p *ProgSpec) numaCtlBinary() string { - return "numactl" -} - -func (p *ProgSpec) prependNumaCtl(args []string) []string { - v := util.NewArrayList[string](5 + len(args)) - v.Append("numactl") - if p.numaSocket >= 0 { - v.Append(fmt.Sprintf("--cpunodebind=%d", p.numaSocket)) - v.Append(fmt.Sprintf("--membind=%d", p.numaSocket)) - } else { - v.Append("--interleave") - v.Append("all") - } - v.AppendAll(args...) - return v -} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/numactl_test.go b/client/go/internal/admin/vespa-wrapper/startcbinary/numactl_test.go deleted file mode 100644 index 65f52be988e..00000000000 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/numactl_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package startcbinary - -import ( - "fmt" - "os" - "runtime" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" -) - -func setup(t *testing.T, testFileName string) { - trace.AdjustVerbosity(1) - mockBinParent = strings.TrimSuffix(testFileName, "/numactl_test.go") - tmpBin = t.TempDir() + "/mock.bin.numactl_test" - err := os.MkdirAll(tmpBin, 0755) - assert.Nil(t, err) - t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH"))) -} - -func TestNumaCtlDetection(t *testing.T) { - if runtime.GOOS == "windows" { - return - } - _, tfn, _, _ := runtime.Caller(0) - setup(t, tfn) - orig := []string{"/bin/myprog", "-c", "cfgid"} - spec := NewProgSpec(orig) - - useMock("no-numactl", "numactl") - spec.configureNumaCtl() - assert.Equal(t, false, spec.shouldUseNumaCtl) - - useMock("bad-numactl", "numactl") - spec.configureNumaCtl() - assert.Equal(t, false, spec.shouldUseNumaCtl) - - t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "") - useMock("good-numactl", "numactl") - spec.configureNumaCtl() - assert.Equal(t, true, spec.shouldUseNumaCtl) - assert.Equal(t, -1, spec.numaSocket) - argv := spec.prependNumaCtl(orig) - trace.Trace("argv:", argv) - assert.Equal(t, 6, len(argv)) - assert.Equal(t, "numactl", argv[0]) - assert.Equal(t, "--interleave", argv[1]) - assert.Equal(t, "all", argv[2]) - assert.Equal(t, "/bin/myprog-bin", argv[3]) - assert.Equal(t, "-c", argv[4]) - assert.Equal(t, "cfgid", argv[5]) - - t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "0") - spec.configureNumaCtl() - assert.Equal(t, true, spec.shouldUseNumaCtl) - assert.Equal(t, 0, spec.numaSocket) - argv = spec.prependNumaCtl(orig) - trace.Trace("argv:", argv) - assert.Equal(t, 6, len(argv)) - assert.Equal(t, "numactl", argv[0]) - assert.Equal(t, "--cpunodebind=0", argv[1]) - assert.Equal(t, "--membind=0", argv[2]) - assert.Equal(t, "/bin/myprog-bin", argv[3]) - assert.Equal(t, "-c", argv[4]) - assert.Equal(t, "cfgid", argv[5]) - - t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "1") - spec.configureNumaCtl() - assert.Equal(t, true, spec.shouldUseNumaCtl) - assert.Equal(t, 1, spec.numaSocket) - argv = spec.prependNumaCtl(orig) - trace.Trace("argv:", argv) - assert.Equal(t, 6, len(argv)) - assert.Equal(t, "numactl", argv[0]) - assert.Equal(t, "--cpunodebind=1", argv[1]) - assert.Equal(t, "--membind=1", argv[2]) - assert.Equal(t, "/bin/myprog-bin", argv[3]) - assert.Equal(t, "-c", argv[4]) - assert.Equal(t, "cfgid", argv[5]) - - t.Setenv("VESPA_AFFINITY_CPU_SOCKET", "2") - spec.configureNumaCtl() - assert.Equal(t, true, spec.shouldUseNumaCtl) - assert.Equal(t, 0, spec.numaSocket) - -} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/progspec.go b/client/go/internal/admin/vespa-wrapper/startcbinary/progspec.go index 9975f6c3c90..b0dcc402893 100644 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/progspec.go +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/progspec.go @@ -8,56 +8,21 @@ import ( "strings" "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/admin/prog" ) -type ProgSpec struct { - Program string - Args []string - BaseName string - Env map[string]string - numaSocket int - shouldUseCallgrind bool - shouldUseValgrind bool - shouldUseNumaCtl bool - shouldUseVespaMalloc bool - vespaMallocPreload string -} - -func NewProgSpec(argv []string) *ProgSpec { +func NewProgSpec(argv []string) *prog.Spec { progName := argv[0] binProg := progName + "-bin" - p := ProgSpec{ - Program: binProg, - Args: argv, - BaseName: baseNameOf(progName), - Env: make(map[string]string), - numaSocket: -1, - } + p := prog.NewSpec(argv) + p.Program = binProg p.Args[0] = binProg - return &p -} - -func baseNameOf(s string) string { - idx := strings.LastIndex(s, "/") - idx++ - return s[idx:] -} - -func (p *ProgSpec) setenv(k, v string) { - p.Env[k] = v -} - -func (p *ProgSpec) getenv(k string) string { - if v, ok := p.Env[k]; ok { - return v - } - return os.Getenv(k) + return p } -func (p *ProgSpec) prependPath(dirName string) { +func prependPath(dirName string, p *prog.Spec) { pathList := []string{dirName} - oldPath := p.getenv(envvars.PATH) + oldPath := p.Getenv(envvars.PATH) if oldPath == "" { oldPath = "/usr/bin" } @@ -67,78 +32,6 @@ func (p *ProgSpec) prependPath(dirName string) { } } newPath := strings.Join(pathList, ":") - p.setenv(envvars.PATH, newPath) + p.Setenv(envvars.PATH, newPath) os.Setenv(envvars.PATH, newPath) } - -func (p *ProgSpec) matchesListEnv(envVarName string) bool { - return p.matchesListString(p.getenv(envVarName)) -} - -func (p *ProgSpec) matchesListString(env string) bool { - if env == "all" { - trace.Debug(p.Program, "always matching in:", env) - return true - } - parts := strings.Fields(env) - for _, part := range parts { - if p.BaseName == part { - trace.Debug(p.Program, "has basename matching in:", env) - return true - } - trace.Debug("checking matching:", p.BaseName, "!=", part) - } - return false -} - -func (p *ProgSpec) valueFromListEnv(envVarName string) string { - return p.valueFromListString(p.getenv(envVarName)) -} - -func (p *ProgSpec) valueFromListString(env string) string { - parts := strings.Fields(env) - for _, part := range parts { - idx := strings.Index(part, "=") - if idx <= 0 { - trace.Trace("expected key=value, but got:", part) - continue - } - partName := part[:idx] - idx++ - partValue := part[idx:] - if p.BaseName == partName || partName == "all" { - trace.Debug(p.Program, "has basename matching in:", env) - return partValue - } - trace.Debug("checking matching:", p.BaseName, "!=", part) - } - return "" -} - -func (spec *ProgSpec) effectiveEnv() []string { - env := make(map[string]string) - for _, entry := range os.Environ() { - addInMap := func(kv string) bool { - for idx, elem := range kv { - if elem == '=' { - k := kv[:idx] - env[k] = kv - return true - } - } - return false - } - if !addInMap(entry) { - env[entry] = "" - } - } - for k, v := range spec.Env { - trace.Trace("add to environment:", k, "=", v) - env[k] = k + "=" + v - } - envv := make([]string, 0, len(env)) - for _, v := range env { - envv = append(envv, v) - } - return envv -} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/progspec_test.go b/client/go/internal/admin/vespa-wrapper/startcbinary/progspec_test.go deleted file mode 100644 index be113e4e350..00000000000 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/progspec_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package startcbinary - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestProgSpec(t *testing.T) { - spec := NewProgSpec([]string{"/opt/vespa/bin/foobar"}) - var b bool - - b = spec.matchesListString("") - assert.Equal(t, false, b) - b = spec.matchesListString("foobar") - assert.Equal(t, true, b) - b = spec.matchesListString("foo bar") - assert.Equal(t, false, b) - b = spec.matchesListString("one foobar") - assert.Equal(t, true, b) - b = spec.matchesListString("foobar two") - assert.Equal(t, true, b) - b = spec.matchesListString("one foobar two") - assert.Equal(t, true, b) - b = spec.matchesListString("all") - assert.Equal(t, true, b) - - var s string - s = spec.valueFromListString("") - assert.Equal(t, "", s) - s = spec.valueFromListString("foobar=123") - assert.Equal(t, "123", s) - s = spec.valueFromListString("one=1 foobar=123 two=2") - assert.Equal(t, "123", s) - s = spec.valueFromListString("one=1 all=123") - assert.Equal(t, "123", s) -} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/startcbinary.go b/client/go/internal/admin/vespa-wrapper/startcbinary/startcbinary.go index f5e58e59808..a062f631b2c 100644 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/startcbinary.go +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/startcbinary.go @@ -7,20 +7,19 @@ import ( "fmt" "os" - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/admin/prog" ) -func startCbinary(spec *ProgSpec) int { - spec.configureCommonEnv() - spec.configurePath() - spec.configureTuning() - spec.configureValgrind() - spec.configureNumaCtl() - spec.configureHugePages() - spec.configureUseMadvise() - spec.configureVespaMalloc() - err := spec.run() +func startCbinary(spec *prog.Spec) int { + configureCommonEnv(spec) + configurePath(spec) + configureTuning() + spec.ConfigureValgrind() + spec.ConfigureNumaCtl() + spec.ConfigureHugePages() + spec.ConfigureUseMadvise() + spec.ConfigureVespaMalloc() + err := spec.Run() if err != nil { fmt.Fprintln(os.Stderr, err) return 1 @@ -28,20 +27,3 @@ func startCbinary(spec *ProgSpec) int { return 0 } } - -func (spec *ProgSpec) run() error { - prog := spec.Program - args := spec.Args - if spec.shouldUseValgrind { - args = spec.prependValgrind(args) - prog = spec.valgrindBinary() - } else if spec.shouldUseNumaCtl { - args = spec.prependNumaCtl(args) - prog = spec.numaCtlBinary() - } - if spec.shouldUseVespaMalloc { - spec.setenv(envvars.LD_PRELOAD, spec.vespaMallocPreload) - } - envv := spec.effectiveEnv() - return util.Execvpe(prog, args, envv) -} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go b/client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go index 57230629d7a..f839d6a0946 100644 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/tuning.go @@ -7,7 +7,7 @@ import ( "github.com/vespa-engine/vespa/client/go/internal/util" ) -func (spec *ProgSpec) configureTuning() { +func configureTuning() { util.OptionallyReduceTimerFrequency() util.TuneResourceLimits() } diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go deleted file mode 100644 index cccb37df8e5..00000000000 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package startcbinary - -import ( - "fmt" - "os" - "strings" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func (p *ProgSpec) configureValgrind() { - p.shouldUseValgrind = false - p.shouldUseCallgrind = false - env := p.getenv(envvars.VESPA_USE_VALGRIND) - allValgrind := env == "all" - parts := strings.Split(env, " ") - for _, part := range parts { - if p.BaseName == part || allValgrind { - trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env) - backticks := util.BackTicksWithStderr - out, err := backticks.Run("which", "valgrind") - if err != nil { - trace.Trace("no valgrind, 'which' fails:", err, "=>", out) - return - } - if opts := p.getenv(envvars.VESPA_VALGRIND_OPT); strings.Contains(opts, "callgrind") { - p.shouldUseCallgrind = true - } - p.shouldUseValgrind = true - return - } - trace.Debug("checking", envvars.VESPA_USE_VALGRIND, ":", p.BaseName, "!=", part) - } -} - -func (p *ProgSpec) valgrindBinary() string { - return "valgrind" -} - -func (p *ProgSpec) valgrindOptions() []string { - env := p.getenv(envvars.VESPA_VALGRIND_OPT) - if env != "" { - return strings.Fields(env) - } - result := []string{ - "--num-callers=32", - "--run-libc-freeres=yes", - "--track-origins=yes", - "--freelist-vol=1000000000", - "--leak-check=full", - "--show-reachable=yes", - } - result = addValgrindSuppression(result, "etc/vespa/valgrind-suppressions.txt") - result = addValgrindSuppression(result, "etc/vespa/suppressions.txt") - return result -} - -func addValgrindSuppression(r []string, fn string) []string { - existsOk, fileName := vespa.HasFileUnderVespaHome(fn) - if existsOk { - r = append(r, fmt.Sprintf("--suppressions=%s", fileName)) - } - return r -} - -func (p *ProgSpec) valgrindLogOption() string { - return fmt.Sprintf("--log-file=%s/tmp/valgrind.%s.log.%d", vespa.FindHome(), p.BaseName, os.Getpid()) -} - -func (p *ProgSpec) prependValgrind(args []string) []string { - v := util.NewArrayList[string](15 + len(args)) - v.Append(p.valgrindBinary()) - v.AppendAll(p.valgrindOptions()...) - v.Append(p.valgrindLogOption()) - v.AppendAll(args...) - return v -} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go deleted file mode 100644 index 1a105d66c4a..00000000000 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package startcbinary - -import ( - "fmt" - "os" - "runtime" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -var tmpBin string -var mockBinParent string - -func useMock(prog, target string) { - mock := fmt.Sprintf("%s/mockbin/%s", mockBinParent, prog) - symlink := fmt.Sprintf("%s/%s", tmpBin, target) - os.Remove(symlink) - err := os.Symlink(mock, symlink) - if err != nil { - util.JustExitWith(err) - } -} - -func setupValgrind(t *testing.T, testFileName string) { - trace.AdjustVerbosity(1) - t.Setenv("VESPA_HOME", mockBinParent+"/mock_vespahome") - mockBinParent = strings.TrimSuffix(testFileName, "/valgrind_test.go") - tmpBin = t.TempDir() + "/mock.bin.valgrind_test" - err := os.MkdirAll(tmpBin, 0755) - assert.Nil(t, err) - t.Setenv("PATH", fmt.Sprintf("%s:%s", tmpBin, os.Getenv("PATH"))) -} - -func TestValgrindDetection(t *testing.T) { - if runtime.GOOS == "windows" { - return - } - _, tfn, _, _ := runtime.Caller(0) - setupValgrind(t, tfn) - spec := NewProgSpec([]string{"/opt/vespa/bin/foobar"}) - var argv []string - - useMock("has-valgrind", "which") - - t.Setenv("VESPA_USE_VALGRIND", "") - spec.configureValgrind() - assert.Equal(t, false, spec.shouldUseValgrind) - assert.Equal(t, false, spec.shouldUseCallgrind) - - t.Setenv("VESPA_USE_VALGRIND", "all") - spec.configureValgrind() - assert.Equal(t, true, spec.shouldUseValgrind) - assert.Equal(t, false, spec.shouldUseCallgrind) - - t.Setenv("VESPA_USE_VALGRIND", "foo bar") - spec.configureValgrind() - assert.Equal(t, false, spec.shouldUseValgrind) - assert.Equal(t, false, spec.shouldUseCallgrind) - - t.Setenv("VESPA_USE_VALGRIND", "foobar") - spec.configureValgrind() - assert.Equal(t, true, spec.shouldUseValgrind) - assert.Equal(t, false, spec.shouldUseCallgrind) - - argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"}) - trace.Trace("argv:", argv) - assert.Equal(t, 11, len(argv)) - assert.Equal(t, "valgrind", argv[0]) - assert.Equal(t, "/bin/myprog", argv[8]) - - t.Setenv("VESPA_USE_VALGRIND", "another foobar yetmore") - spec.configureValgrind() - assert.Equal(t, true, spec.shouldUseValgrind) - assert.Equal(t, false, spec.shouldUseCallgrind) - - t.Setenv("VESPA_VALGRIND_OPT", "--tool=callgrind") - spec.configureValgrind() - assert.Equal(t, true, spec.shouldUseValgrind) - assert.Equal(t, true, spec.shouldUseCallgrind) - - argv = spec.prependValgrind([]string{"/bin/myprog", "-c", "cfgid"}) - trace.Trace("argv:", argv) - assert.Equal(t, 6, len(argv)) - assert.Equal(t, "valgrind", argv[0]) - assert.Equal(t, "--tool=callgrind", argv[1]) - assert.Equal(t, "/bin/myprog", argv[3]) - - useMock("no-valgrind", "which") - spec.configureValgrind() - assert.Equal(t, false, spec.shouldUseValgrind) - assert.Equal(t, false, spec.shouldUseCallgrind) -} diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/vespamalloc.go b/client/go/internal/admin/vespa-wrapper/startcbinary/vespamalloc.go deleted file mode 100644 index c6d53e1d03c..00000000000 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/vespamalloc.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Author: arnej - -package startcbinary - -import ( - "fmt" - - "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" - "github.com/vespa-engine/vespa/client/go/internal/admin/trace" - "github.com/vespa-engine/vespa/client/go/internal/vespa" -) - -func vespaMallocLib(suf string) string { - prefixes := []string{"lib64", "lib"} - for _, pre := range prefixes { - fn := fmt.Sprintf("%s/vespa/malloc/%s", pre, suf) - existsOk, fileName := vespa.HasFileUnderVespaHome(fn) - if existsOk { - trace.Debug("found library:", fileName) - return fileName - } - trace.Debug("bad or missing library:", fn) - } - return "" -} - -func (p *ProgSpec) configureVespaMalloc() { - p.shouldUseVespaMalloc = false - if p.matchesListEnv(envvars.VESPA_USE_NO_VESPAMALLOC) { - trace.Trace("use no vespamalloc:", p.BaseName) - return - } - if p.shouldUseValgrind && !p.shouldUseCallgrind { - trace.Trace("use valgrind, so no vespamalloc:", p.BaseName) - return - } - var useFile string - if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_DST) { - useFile = vespaMallocLib("libvespamallocdst16.so") - } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC_D) { - useFile = vespaMallocLib("libvespamallocd.so") - } else if p.matchesListEnv(envvars.VESPA_USE_VESPAMALLOC) { - useFile = vespaMallocLib("libvespamalloc.so") - } - trace.Trace("use file:", useFile) - if useFile == "" { - return - } - if loadAsHuge := p.getenv(envvars.VESPA_LOAD_CODE_AS_HUGEPAGES); loadAsHuge != "" { - otherFile := vespaMallocLib("libvespa_load_as_huge.so") - useFile = fmt.Sprintf("%s:%s", useFile, otherFile) - } - p.considerEnvFallback(envvars.VESPA_MALLOC_HUGEPAGES, envvars.VESPA_USE_HUGEPAGES) - p.vespaMallocPreload = useFile - p.shouldUseVespaMalloc = true -} diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go index 06568dd35c3..a6447ef8d2e 100644 --- a/client/go/internal/cli/cmd/feed.go +++ b/client/go/internal/cli/cmd/feed.go @@ -18,10 +18,11 @@ import ( func addFeedFlags(cmd *cobra.Command, options *feedOptions) { cmd.PersistentFlags().IntVar(&options.connections, "connections", 8, "The number of connections to use") cmd.PersistentFlags().StringVar(&options.compression, "compression", "auto", `Compression mode to use. Default is "auto" which compresses large documents. Must be "auto", "gzip" or "none"`) + cmd.PersistentFlags().IntVar(&options.timeoutSecs, "timeout", 0, "Invididual feed operation timeout in seconds. 0 to disable") + cmd.PersistentFlags().IntVar(&options.doomSecs, "max-failure-seconds", 0, "Exit if given number of seconds elapse without any successful operations. 0 to disable") + cmd.PersistentFlags().BoolVar(&options.verbose, "verbose", false, "Verbose mode. Print successful operations in addition to errors") cmd.PersistentFlags().StringVar(&options.route, "route", "", "Target Vespa route for feed operations") cmd.PersistentFlags().IntVar(&options.traceLevel, "trace", 0, "The trace level of network traffic. 0 to disable") - cmd.PersistentFlags().IntVar(&options.timeoutSecs, "timeout", 0, "Feed operation timeout in seconds. 0 to disable") - cmd.PersistentFlags().BoolVar(&options.verbose, "verbose", false, "Verbose mode. Print successful operations in addition to errors") memprofile := "memprofile" cpuprofile := "cpuprofile" cmd.PersistentFlags().StringVar(&options.memprofile, memprofile, "", "Write a heap profile to given file") @@ -38,44 +39,34 @@ type feedOptions struct { verbose bool traceLevel int timeoutSecs int - memprofile string - cpuprofile string + doomSecs int + + memprofile string + cpuprofile string } func newFeedCmd(cli *CLI) *cobra.Command { var options feedOptions cmd := &cobra.Command{ - Use: "feed FILE", + Use: "feed FILE [FILE]...", Short: "Feed documents to a Vespa cluster", Long: `Feed documents to a Vespa cluster. -A high performance feeding client. This can be used to feed large amounts of -documents to a Vespa cluster efficiently. +This command can be used to feed large amounts of documents to a Vespa cluster +efficiently. The contents of FILE must be either a JSON array or JSON objects separated by newline (JSONL). If FILE is a single dash ('-'), documents will be read from standard input. `, - Example: `$ vespa feed documents.jsonl -$ cat documents.jsonl | vespa feed - -`, - Args: cobra.ExactArgs(1), + Example: `$ vespa feed docs.jsonl moredocs.json +$ cat docs.jsonl | vespa feed -`, + Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, SilenceUsage: true, Hidden: true, // TODO(mpolden): Remove when ready for public use RunE: func(cmd *cobra.Command, args []string) error { - var r io.Reader - if args[0] == "-" { - r = cli.Stdin - } else { - f, err := os.Open(args[0]) - if err != nil { - return err - } - defer f.Close() - r = f - } if options.cpuprofile != "" { f, err := os.Create(options.cpuprofile) if err != nil { @@ -84,7 +75,7 @@ $ cat documents.jsonl | vespa feed - pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } - err := feed(r, cli, options) + err := feed(args, options, cli) if options.memprofile != "" { f, err := os.Create(options.memprofile) if err != nil { @@ -123,7 +114,7 @@ func (opts feedOptions) compressionMode() (document.Compression, error) { return 0, errHint(fmt.Errorf("invalid compression mode: %s", opts.compression), `Must be "auto", "gzip" or "none"`) } -func feed(r io.Reader, cli *CLI, options feedOptions) error { +func feed(files []string, options feedOptions, cli *CLI) error { service, err := documentService(cli) if err != nil { return err @@ -139,25 +130,37 @@ func feed(r io.Reader, cli *CLI, options feedOptions) error { Route: options.route, TraceLevel: options.traceLevel, BaseURL: service.BaseURL, + NowFunc: cli.now, }, clients) throttler := document.NewThrottler(options.connections) - // TODO(mpolden): Make doom duration configurable - circuitBreaker := document.NewCircuitBreaker(10*time.Second, 0) + circuitBreaker := document.NewCircuitBreaker(10*time.Second, time.Duration(options.doomSecs)*time.Second) dispatcher := document.NewDispatcher(client, throttler, circuitBreaker, cli.Stderr, options.verbose) - dec := document.NewDecoder(r) - start := cli.now() - for { - doc, err := dec.Decode() - if err == io.EOF { - break - } - if err != nil { - cli.printErr(fmt.Errorf("failed to decode document: %w", err)) + for _, name := range files { + var r io.ReadCloser + if len(files) == 1 && name == "-" { + r = io.NopCloser(cli.Stdin) + } else { + f, err := os.Open(name) + if err != nil { + return err + } + r = f } - if err := dispatcher.Enqueue(doc); err != nil { - cli.printErr(err) + dec := document.NewDecoder(r) + for { + doc, err := dec.Decode() + if err == io.EOF { + break + } + if err != nil { + cli.printErr(fmt.Errorf("failed to decode document: %w", err)) + } + if err := dispatcher.Enqueue(doc); err != nil { + cli.printErr(err) + } } + r.Close() } if err := dispatcher.Close(); err != nil { return err diff --git a/client/go/internal/cli/cmd/feed_test.go b/client/go/internal/cli/cmd/feed_test.go index 521d2b2abd0..467d55a0a6e 100644 --- a/client/go/internal/cli/cmd/feed_test.go +++ b/client/go/internal/cli/cmd/feed_test.go @@ -31,47 +31,50 @@ func TestFeed(t *testing.T) { cli.now = clock.now td := t.TempDir() - jsonFile := filepath.Join(td, "docs.jsonl") - err := os.WriteFile(jsonFile, []byte(`{ + doc := []byte(`{ "put": "id:ns:type::doc1", "fields": {"foo": "123"} -}`), 0644) - - require.Nil(t, err) +}`) + jsonFile1 := filepath.Join(td, "docs1.jsonl") + jsonFile2 := filepath.Join(td, "docs2.jsonl") + require.Nil(t, os.WriteFile(jsonFile1, doc, 0644)) + require.Nil(t, os.WriteFile(jsonFile2, doc, 0644)) httpClient.NextResponseString(200, `{"message":"OK"}`) - require.Nil(t, cli.Run("feed", jsonFile)) + httpClient.NextResponseString(200, `{"message":"OK"}`) + require.Nil(t, cli.Run("feed", jsonFile1, jsonFile2)) assert.Equal(t, "", stderr.String()) want := `{ - "feeder.seconds": 1.000, - "feeder.ok.count": 1, - "feeder.ok.rate": 1.000, + "feeder.seconds": 5.000, + "feeder.ok.count": 2, + "feeder.ok.rate": 0.400, "feeder.error.count": 0, "feeder.inflight.count": 0, - "http.request.count": 1, - "http.request.bytes": 25, + "http.request.count": 2, + "http.request.bytes": 50, "http.request.MBps": 0.000, "http.exception.count": 0, - "http.response.count": 1, - "http.response.bytes": 16, + "http.response.count": 2, + "http.response.bytes": 32, "http.response.MBps": 0.000, "http.response.error.count": 0, - "http.response.latency.millis.min": 0, - "http.response.latency.millis.avg": 0, - "http.response.latency.millis.max": 0, + "http.response.latency.millis.min": 1000, + "http.response.latency.millis.avg": 1000, + "http.response.latency.millis.max": 1000, "http.response.code.counts": { - "200": 1 + "200": 2 } } ` assert.Equal(t, want, stdout.String()) stdout.Reset() - cli.Stdin = bytes.NewBuffer([]byte(`{ - "put": "id:ns:type::doc1", - "fields": {"foo": "123"} -}`)) + var stdinBuf bytes.Buffer + stdinBuf.Write(doc) + stdinBuf.Write(doc) + cli.Stdin = &stdinBuf + httpClient.NextResponseString(200, `{"message":"OK"}`) httpClient.NextResponseString(200, `{"message":"OK"}`) require.Nil(t, cli.Run("feed", "-")) assert.Equal(t, want, stdout.String()) diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go index 6a5b936434c..10fb2743c63 100644 --- a/client/go/internal/cli/cmd/visit.go +++ b/client/go/internal/cli/cmd/visit.go @@ -33,6 +33,8 @@ type visitArgs struct { to string slices int sliceId int + bucketSpace string + bucketSpaces []string cli *CLI } @@ -132,6 +134,7 @@ $ vespa visit --field-set "[id]" # list document IDs cmd.Flags().StringVar(&vArgs.to, "to", "", `Timestamp to visit up to, in seconds`) cmd.Flags().IntVar(&vArgs.sliceId, "slice-id", -1, `The number of the slice this visit invocation should fetch`) cmd.Flags().IntVar(&vArgs.slices, "slices", -1, `Split the document corpus into this number of independent slices`) + cmd.Flags().StringSliceVar(&vArgs.bucketSpaces, "bucket-space", []string{"global", "default"}, `"default" or "global" bucket space`) return cmd } @@ -157,6 +160,16 @@ func checkArguments(vArgs visitArgs) (res util.OperationResult) { return util.Failure("Invalid 'to' argument: '" + vArgs.to + "': " + err.Error()) } } + for _, b := range vArgs.bucketSpaces { + switch b { + case + "default", + "global": + // Do nothing + default: + return util.Failure("Invalid 'bucket-space' argument '" + b + "', must be 'default' or 'global'") + } + } return util.Success("") } @@ -226,13 +239,16 @@ func visitClusters(vArgs *visitArgs, service *vespa.Service) (res util.Operation if vArgs.makeFeed { vArgs.writeString("[\n") } - for _, c := range clusters { - vArgs.contentCluster = c - res = runVisit(vArgs, service) - if !res.Success { - return res + for _, b := range vArgs.bucketSpaces { + for _, c := range clusters { + vArgs.bucketSpace = b + vArgs.contentCluster = c + res = runVisit(vArgs, service) + if !res.Success { + return res + } + vArgs.debugPrint("Success: " + res.Message) } - vArgs.debugPrint("Success: " + res.Message) } if vArgs.makeFeed { vArgs.writeString("{}\n]\n") @@ -330,6 +346,9 @@ func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*V if vArgs.slices > 0 { urlPath = urlPath + fmt.Sprintf("&slices=%d&sliceId=%d", vArgs.slices, vArgs.sliceId) } + if vArgs.bucketSpace != "" { + urlPath = urlPath + "&bucketSpace=" + vArgs.bucketSpace + } url, urlParseError := url.Parse(urlPath) if urlParseError != nil { return nil, util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go index b6e5b893e0b..9bb8f61554a 100644 --- a/client/go/internal/cli/cmd/visit_test.go +++ b/client/go/internal/cli/cmd/visit_test.go @@ -105,6 +105,7 @@ func TestVisitCommand(t *testing.T) { assertVisitResults( []string{ "visit", + "--bucket-space", "default", "--json-lines", }, t, @@ -118,7 +119,7 @@ func TestVisitCommand(t *testing.T) { document3 + `],"documentCount":2}`, }, - "cluster=fooCC&continuation=CAFE&wantedDocumentCount=1000", + "cluster=fooCC&continuation=CAFE&wantedDocumentCount=1000&bucketSpace=default", document1+"\n"+ document2+"\n"+ document3+"\n") diff --git a/client/go/internal/util/http.go b/client/go/internal/util/http.go index 8a67b24dffb..30874153510 100644 --- a/client/go/internal/util/http.go +++ b/client/go/internal/util/http.go @@ -81,8 +81,9 @@ func ForceHTTP2(client HTTPClient, certificates []tls.Certificate, caCertificate // https://github.com/golang/go/issues/16582 // https://github.com/golang/go/issues/22091 c.client.Transport = &http2.Transport{ - AllowHTTP: true, - DialTLSContext: dialFunc, + AllowHTTP: true, + DialTLSContext: dialFunc, + StrictMaxConcurrentStreams: true, } ConfigureTLS(client, certificates, caCertificate, trustAll) } diff --git a/client/go/internal/vespa/document/dispatcher.go b/client/go/internal/vespa/document/dispatcher.go index 5c99f3bf056..b87dfaf55eb 100644 --- a/client/go/internal/vespa/document/dispatcher.go +++ b/client/go/internal/vespa/document/dispatcher.go @@ -20,7 +20,6 @@ type Dispatcher struct { stats Stats started bool - ready chan Id results chan Result msgs chan string @@ -128,7 +127,6 @@ func (d *Dispatcher) start() { return } d.listPool.New = func() any { return list.New() } - d.ready = make(chan Id, 4096) d.results = make(chan Result, 4096) d.msgs = make(chan string, 4096) d.started = true @@ -164,27 +162,35 @@ func (d *Dispatcher) enqueue(op documentOp) error { } d.mu.Unlock() group.add(op, op.attempts > 0) - d.enqueueWithSlot(op.document.Id) + d.dispatch(op.document.Id, group) return nil } -func (d *Dispatcher) enqueueWithSlot(id Id) { +func (d *Dispatcher) dispatch(id Id, group *documentGroup) { + if !d.canDispatch() { + d.msgs <- fmt.Sprintf("refusing to dispatch document %s: too many errors", id) + return + } d.acquireSlot() - d.ready <- id - d.throttler.Sent() - d.dispatch() -} - -func (d *Dispatcher) dispatch() { d.workerWg.Add(1) go func() { defer d.workerWg.Done() - id := <-d.ready - d.mu.RLock() - group := d.inflight[id.String()] - d.mu.RUnlock() d.sendDocumentIn(group) }() + d.throttler.Sent() +} + +func (d *Dispatcher) canDispatch() bool { + switch d.circuitBreaker.State() { + case CircuitClosed: + return true + case CircuitHalfOpen: + time.Sleep(time.Second) + return true + case CircuitOpen: + return false + } + panic("invalid circuit state") } func (d *Dispatcher) acquireSlot() { diff --git a/client/go/internal/vespa/document/dispatcher_test.go b/client/go/internal/vespa/document/dispatcher_test.go index d066f5bc9ae..2e2e9a5abbd 100644 --- a/client/go/internal/vespa/document/dispatcher_test.go +++ b/client/go/internal/vespa/document/dispatcher_test.go @@ -36,6 +36,12 @@ func (f *mockFeeder) Send(doc Document) Result { return result } +type mockCircuitBreaker struct{ state CircuitState } + +func (c *mockCircuitBreaker) Success() {} +func (c *mockCircuitBreaker) Error(err error) {} +func (c *mockCircuitBreaker) State() CircuitState { return c.state } + func TestDispatcher(t *testing.T) { feeder := &mockFeeder{} clock := &manualClock{tick: time.Second} @@ -130,3 +136,32 @@ func TestDispatcherOrderingWithFailures(t *testing.T) { assert.Equal(t, int64(2), dispatcher.Stats().Errors) assert.Equal(t, 6, len(feeder.documents)) } + +func TestDispatcherOpenCircuit(t *testing.T) { + feeder := &mockFeeder{} + doc := Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut} + clock := &manualClock{tick: time.Second} + throttler := newThrottler(8, clock.now) + breaker := &mockCircuitBreaker{} + dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard, false) + dispatcher.Enqueue(doc) + breaker.state = CircuitOpen + dispatcher.Enqueue(doc) + dispatcher.Close() + assert.Equal(t, 1, len(feeder.documents)) +} + +func BenchmarkDocumentDispatching(b *testing.B) { + feeder := &mockFeeder{} + clock := &manualClock{tick: time.Second} + throttler := newThrottler(8, clock.now) + breaker := NewCircuitBreaker(time.Second, 0) + dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard, false) + doc := Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields":{"foo": "123"}}`)} + b.ResetTimer() // ignore setup time + + for n := 0; n < b.N; n++ { + dispatcher.enqueue(documentOp{document: doc}) + dispatcher.workerWg.Wait() + } +} diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index 51b6fa4de39..877bcc5edce 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -11,6 +11,7 @@ import ( "net/url" "strconv" "strings" + "sync" "sync/atomic" "time" @@ -31,6 +32,7 @@ type Client struct { httpClients []countingHTTPClient now func() time.Time sendCount int32 + gzippers sync.Pool } // ClientOptions specifices the configuration options of a feed client. @@ -40,6 +42,7 @@ type ClientOptions struct { Route string TraceLevel int Compression Compression + NowFunc func() time.Time } type countingHTTPClient struct { @@ -73,11 +76,17 @@ func NewClient(options ClientOptions, httpClients []util.HTTPClient) *Client { for _, client := range httpClients { countingClients = append(countingClients, countingHTTPClient{client: client}) } - return &Client{ + nowFunc := options.NowFunc + if nowFunc == nil { + nowFunc = time.Now + } + c := &Client{ options: options, httpClients: countingClients, - now: time.Now, + now: nowFunc, } + c.gzippers.New = func() any { return gzip.NewWriter(io.Discard) } + return c } func (c *Client) queryParams() url.Values { @@ -162,18 +171,25 @@ func (c *Client) leastBusyClient() *countingHTTPClient { return &leastBusy } +func (c *Client) gzipWriter(w io.Writer) *gzip.Writer { + gzipWriter := c.gzippers.Get().(*gzip.Writer) + gzipWriter.Reset(w) + return gzipWriter +} + func (c *Client) createRequest(method, url string, body []byte) (*http.Request, error) { var r io.Reader useGzip := c.options.Compression == CompressionGzip || (c.options.Compression == CompressionAuto && len(body) > 512) if useGzip { var buf bytes.Buffer - w := gzip.NewWriter(&buf) + w := c.gzipWriter(&buf) if _, err := w.Write(body); err != nil { return nil, err } if err := w.Close(); err != nil { return nil, err } + c.gzippers.Put(w) r = &buf } else { r = bytes.NewReader(body) diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index 314113c53be..f67368b5128 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -293,3 +293,26 @@ func TestClientFeedURL(t *testing.T) { } } } + +func benchmarkClientSend(b *testing.B, compression Compression, document Document) { + httpClient := mock.HTTPClient{} + client := NewClient(ClientOptions{ + Compression: compression, + BaseURL: "https://example.com:1337", + Timeout: time.Duration(5 * time.Second), + }, []util.HTTPClient{&httpClient}) + b.ResetTimer() // ignore setup + for n := 0; n < b.N; n++ { + client.Send(document) + } +} + +func BenchmarkClientSend(b *testing.B) { + doc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "my document"}}`)} + benchmarkClientSend(b, CompressionNone, doc) +} + +func BenchmarkClientSendCompressed(b *testing.B) { + doc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "my document"}}`)} + benchmarkClientSend(b, CompressionGzip, doc) +} diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 69e1a94a813..7971f4eebcf 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -45,7 +45,7 @@ <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>11.0.14</jetty.version> + <jetty.version>11.0.15</jetty.version> <org.lz4.version>1.8.0</org.lz4.version> <org.json.version>20230227</org.json.version> <!-- TODO: Remove on Vespa 9 --> <slf4j.version>1.7.32</slf4j.version> <!-- WARNING: when updated, also update c.y.v.tenant:base pom --> diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 4fb61ed8b5e..4306744eb20 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -213,7 +213,7 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter public <T extends ConfigModel> List<T> getModels(Class<T> modelClass) { List<T> modelsOfModelClass = new ArrayList<>(); - for (ConfigModel model : asMap().values()) { + for (ConfigModel model : configModels) { if (modelClass.isInstance(model)) modelsOfModelClass.add((T)model); } diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index 180c8e6012f..93bec4975a6 100644 --- a/config-model/src/main/java/com/yahoo/schema/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -709,8 +709,17 @@ public class Schema implements ImmutableSchema { public FieldSets fieldSets() { return fieldSets; } + private Schema inheritedSchema = null; + + public void setInheritedSchema(Schema value) { + inheritedSchema = value; + } + /** Returns the schema inherited by this, or throws if none */ - private Schema requireInherited() { return owner.schemas().get(inherited.get()); } + private Schema requireInherited() { + if (inheritedSchema != null) return inheritedSchema; + return owner.schemas().get(inherited.get()); + } /** * For adding structs defined in document scope diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index 0abcc9e890a..40ec84ec8bc 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -98,6 +98,13 @@ public class ConvertParsedSchemas { Schema schema = parsed.getDocumentWithoutSchema() ? new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties) : new Schema(parsed.name(), applicationPackage, inherited, fileRegistry, deployLogger, properties); + inherited.ifPresent(parentName -> { + for (var possibleParent : resultList) { + if (possibleParent.getName().equals(parentName)) { + schema.setInheritedSchema(possibleParent); + } + } + }); convertSchema(schema, parsed); resultList.add(schema); } @@ -145,7 +152,23 @@ public class ConvertParsedSchemas { docsum.setOmitSummaryFeatures(true); } for (var parsedField : parsed.getSummaryFields()) { - DataType dataType = typeContext.resolveType(parsedField.getType()); + var parsedType = parsedField.getType(); + DataType dataType = (parsedType != null) ? typeContext.resolveType(parsedType) : null; + var existingField = schema.getField(parsedField.name()); + if (existingField != null) { + var existingType = existingField.getDataType(); + if (dataType == null) { + dataType = existingType; + } else if (!dataType.equals(existingType)) { + if (dataType.getValueClass().equals(com.yahoo.document.datatypes.WeightedSet.class)) { + // "adjusting type for field " + parsedField.name() + " in document-summary " + parsed.name() + " field already has: " + existingType + " but declared type was: " + dataType + dataType = existingType; + } + } + } + if (dataType == null) { + throw new IllegalArgumentException("Missing data-type for summary field " + parsedField.name() + " in document-summary " + parsed.name()); + } var summaryField = new SummaryField(parsedField.name(), dataType); // XXX does not belong here: summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java index 6284c0bc625..beb96ab8cc8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java @@ -14,24 +14,30 @@ public class LogForwarder extends AbstractService implements LogforwarderConfig. public final String clientName; public final String splunkHome; public final Integer phoneHomeInterval; + public final String role; - private Config(String ds, String cn, String sh, Integer phi) { + private Config(String ds, String cn, String sh, Integer phi, String role) { this.deploymentServer = ds; this.clientName = cn; this.splunkHome = sh; this.phoneHomeInterval = phi; + this.role = role; } public Config withDeploymentServer(String ds) { - return new Config(ds, clientName, splunkHome, phoneHomeInterval); + return new Config(ds, clientName, splunkHome, phoneHomeInterval, role); } public Config withClientName(String cn) { - return new Config(deploymentServer, cn, splunkHome, phoneHomeInterval); + return new Config(deploymentServer, cn, splunkHome, phoneHomeInterval, role); } public Config withSplunkHome(String sh) { - return new Config(deploymentServer, clientName, sh, phoneHomeInterval); + return new Config(deploymentServer, clientName, sh, phoneHomeInterval, role); } public Config withPhoneHomeInterval(Integer phi) { - return new Config(deploymentServer, clientName, splunkHome, phi); + return new Config(deploymentServer, clientName, splunkHome, phi, role); + } + + public Config withRole(String role) { + return new Config(deploymentServer, clientName, splunkHome, phoneHomeInterval, role); } } @@ -49,7 +55,7 @@ public class LogForwarder extends AbstractService implements LogforwarderConfig. } public static Config cfg() { - return new Config(null, null, null, null); + return new Config(null, null, null, null, null); } // LogForwarder does not need any ports. @@ -79,6 +85,9 @@ public class LogForwarder extends AbstractService implements LogforwarderConfig. if (config.phoneHomeInterval != null) { builder.phoneHomeInterval(config.phoneHomeInterval); } + if (config.role != null) { + builder.role(config.role); + } } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 9280f0ceb9a..df998e75268 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -21,9 +21,11 @@ import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets; import com.yahoo.vespa.model.admin.monitoring.builder.xml.MetricsBuilder; import org.w3c.dom.Element; + import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.regex.Pattern; /** * A base class for admin model builders, to support common functionality across versions. @@ -98,7 +100,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu return Optional.empty(); } - void addLogForwarders(ModelElement logForwardingElement, Admin admin) { + void addLogForwarders(ModelElement logForwardingElement, Admin admin, DeployState deployState) { if (logForwardingElement == null) return; boolean alsoForAdminCluster = logForwardingElement.booleanAttribute("include-admin"); for (ModelElement e : logForwardingElement.children("splunk")) { @@ -106,7 +108,8 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu .withSplunkHome(e.stringAttribute("splunk-home")) .withDeploymentServer(e.stringAttribute("deployment-server")) .withClientName(e.stringAttribute("client-name")) - .withPhoneHomeInterval(e.integerAttribute("phone-home-interval")); + .withPhoneHomeInterval(e.integerAttribute("phone-home-interval")) + .withRole(parseLogforwarderRole(e.stringAttribute("role"), deployState)); admin.setLogForwarderConfig(cfg, alsoForAdminCluster); } } @@ -130,4 +133,22 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu } } + private String parseLogforwarderRole(String role, DeployState deployState) { + if (role == null) + return null; + if (deployState.zone().system().isPublic()) + throw new IllegalArgumentException("Logforwarder role not supported in public systems"); + + // Currently only support athenz roles on format athenz://<domain>/role/<role> + var rolePattern = Pattern.compile("(?<scheme>athenz)://" + + "(?<domain>[a-zA-Z0-9_][a-zA-Z0-9_.-]*[a-zA-Z0-9_])" + + "/role/" + + "(?<role>[a-zA-Z0-9_][a-zA-Z0-9_.-]*[a-zA-Z0-9_])"); + var matcher = rolePattern.matcher(role); + if (!matcher.matches()) { + throw new IllegalArgumentException("Invalid role path " + role); + } + return matcher.group("domain") + ":role." + matcher.group("role"); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 7a7092b04dd..152f7e03a4c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -47,7 +47,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { if ( ! admin.multitenant()) admin.setClusterControllers(addConfiguredClusterControllers(deployState, admin, adminE), deployState); - addLogForwarders(new ModelElement(adminE).child("logforwarding"), admin); + addLogForwarders(new ModelElement(adminE).child("logforwarding"), admin, deployState); addLoggingSpecs(new ModelElement(adminE).child("logging"), admin); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 80000e54b1b..4990ddc9a53 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -55,7 +55,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { assignSlobroks(deployState, requestedSlobroks.orElse(NodesSpecification.nonDedicated(3, context)), admin); assignLogserver(deployState, requestedLogservers.orElse(createNodesSpecificationForLogserver()), admin); - addLogForwarders(adminElement.child("logforwarding"), admin); + addLogForwarders(adminElement.child("logforwarding"), admin, deployState); addLoggingSpecs(adminElement.child("logging"), admin); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index 5a96e33c522..e2166b263ee 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.utils.Duration; import org.w3c.dom.Element; import java.util.Optional; @@ -47,9 +48,29 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc clusterControllerTuning = tuning.child("cluster-controller"); } + var tuningConfig = new ClusterControllerTuningBuilder(clusterControllerTuning, + minNodeRatioPerGroup, + bucketSplittingMinimumBits) + .build(); + if (ancestor instanceof ContentCluster) { + int numberOfLeafGroups = ((ContentCluster) ancestor).getRootGroup().getNumberOfLeafGroups(); + if (tuningConfig.maxGroupsAllowedDown().isPresent()) { + Integer maxGroupsAllowedDown = tuningConfig.maxGroupsAllowedDown().get(); + if (deployState.zone().environment().isProduction() && (maxGroupsAllowedDown > numberOfLeafGroups)) + throw new IllegalArgumentException("Cannot set max-groups-allowed-down (" + maxGroupsAllowedDown + + ") larger than number of groups (" + numberOfLeafGroups + ")"); + } else { + // Reduce to numberOfLeafGroups for tests or in environments where number of groups are reduced by policy (dev, test, staging, perf) + tuningConfig = tuningConfig.withMaxGroupsAllowedDown(numberOfLeafGroups); + } + } else { + // Reduce to 1 for tests (ancestor is a mock class) + tuningConfig = tuningConfig.withMaxGroupsAllowedDown(1); + } + return new ClusterControllerConfig(ancestor, clusterName, - new ClusterControllerTuning(clusterControllerTuning, minNodeRatioPerGroup, bucketSplittingMinimumBits), + tuningConfig, resourceLimits, allowMoreThanOneContentGroupDown); } @@ -99,45 +120,81 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc resourceLimits.getConfig(builder); } - public ClusterControllerTuning tuning() { return tuning; } - - public static class ClusterControllerTuning { - - private final Optional<Double> minNodeRatioPerGroup; - private final Optional<Duration> initProgressTime; - private final Optional<Duration> transitionTime; - private final Optional<Long> maxPrematureCrashes; - private final Optional<Duration> stableStateTimePeriod; - private final Optional<Double> minDistributorUpRatio; - private final Optional<Double> minStorageUpRatio; - private final Optional<Integer> minSplitBits; - final Optional<Integer> maxGroupsAllowedDown; - - ClusterControllerTuning(ModelElement tuning, - Optional<Double> minNodeRatioPerGroup, - Optional<Integer> bucketSplittingMinimumBits) { - this.minSplitBits = bucketSplittingMinimumBits; - this.minNodeRatioPerGroup = minNodeRatioPerGroup; - if (tuning == null) { - this.initProgressTime = Optional.empty(); - this.transitionTime = Optional.empty(); - this.maxPrematureCrashes = Optional.empty(); - this.stableStateTimePeriod = Optional.empty(); - this.minDistributorUpRatio = Optional.empty(); - this.minStorageUpRatio = Optional.empty(); - this.maxGroupsAllowedDown = Optional.empty(); - } else { - this.initProgressTime = Optional.ofNullable(tuning.childAsDuration("init-progress-time")); - this.transitionTime = Optional.ofNullable(tuning.childAsDuration("transition-time")); - this.maxPrematureCrashes = Optional.ofNullable(tuning.childAsLong("max-premature-crashes")); - this.stableStateTimePeriod = Optional.ofNullable(tuning.childAsDuration("stable-state-period")); - this.minDistributorUpRatio = Optional.ofNullable(tuning.childAsDouble("min-distributor-up-ratio")); - this.minStorageUpRatio = Optional.ofNullable(tuning.childAsDouble("min-storage-up-ratio")); - this.maxGroupsAllowedDown = Optional.ofNullable(tuning.childAsInteger("max-groups-allowed-down")); - } + public ClusterControllerTuning tuning() {return tuning;} + +private static class ClusterControllerTuningBuilder { + + private final Optional<Double> minNodeRatioPerGroup; + private final Optional<Duration> initProgressTime; + private final Optional<Duration> transitionTime; + private final Optional<Long> maxPrematureCrashes; + private final Optional<Duration> stableStateTimePeriod; + private final Optional<Double> minDistributorUpRatio; + private final Optional<Double> minStorageUpRatio; + private final Optional<Integer> minSplitBits; + final Optional<Integer> maxGroupsAllowedDown; + + ClusterControllerTuningBuilder(ModelElement tuning, + Optional<Double> minNodeRatioPerGroup, + Optional<Integer> bucketSplittingMinimumBits) { + this.minSplitBits = bucketSplittingMinimumBits; + this.minNodeRatioPerGroup = minNodeRatioPerGroup; + if (tuning == null) { + this.initProgressTime = Optional.empty(); + this.transitionTime = Optional.empty(); + this.maxPrematureCrashes = Optional.empty(); + this.stableStateTimePeriod = Optional.empty(); + this.minDistributorUpRatio = Optional.empty(); + this.minStorageUpRatio = Optional.empty(); + this.maxGroupsAllowedDown = Optional.empty(); + } else { + this.initProgressTime = Optional.ofNullable(tuning.childAsDuration("init-progress-time")); + this.transitionTime = Optional.ofNullable(tuning.childAsDuration("transition-time")); + this.maxPrematureCrashes = Optional.ofNullable(tuning.childAsLong("max-premature-crashes")); + this.stableStateTimePeriod = Optional.ofNullable(tuning.childAsDuration("stable-state-period")); + this.minDistributorUpRatio = Optional.ofNullable(tuning.childAsDouble("min-distributor-up-ratio")); + this.minStorageUpRatio = Optional.ofNullable(tuning.childAsDouble("min-storage-up-ratio")); + this.maxGroupsAllowedDown = Optional.ofNullable(tuning.childAsInteger("max-groups-allowed-down")); } + } - public Optional<Integer> maxGroupsAllowedDown() { return maxGroupsAllowedDown; } + private ClusterControllerTuning build() { + return new ClusterControllerTuning(initProgressTime, + transitionTime, + maxPrematureCrashes, + stableStateTimePeriod, + minDistributorUpRatio, + minStorageUpRatio, + maxGroupsAllowedDown, + minNodeRatioPerGroup, + minSplitBits); } } + +private record ClusterControllerTuning(Optional<Duration> initProgressTime, + Optional<Duration> transitionTime, + Optional<Long> maxPrematureCrashes, + Optional<Duration> stableStateTimePeriod, + Optional<Double> minDistributorUpRatio, + Optional<Double> minStorageUpRatio, + Optional<Integer> maxGroupsAllowedDown, + Optional<Double> minNodeRatioPerGroup, + Optional<Integer> minSplitBits) { + + public ClusterControllerTuning withMaxGroupsAllowedDown(int maxGroupsAllowedDown) { + return new ClusterControllerConfig.ClusterControllerTuning( + initProgressTime, + transitionTime, + maxPrematureCrashes, + stableStateTimePeriod, + minDistributorUpRatio, + minStorageUpRatio, + Optional.of(maxGroupsAllowedDown), + minNodeRatioPerGroup, + minSplitBits); + } + +} + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index f1f210b013c..f1d5c7c9220 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -164,18 +164,12 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem DeployState deployState, ContentCluster c, ClusterResourceLimits resourceLimits) { - var config = new ClusterControllerConfig.Builder(c.clusterId, - contentElement, - resourceLimits.getClusterControllerLimits(), - deployState.featureFlags() - .allowMoreThanOneContentGroupDown(new ClusterSpec.Id(c.clusterId))) + return new ClusterControllerConfig.Builder(c.clusterId, + contentElement, + resourceLimits.getClusterControllerLimits(), + deployState.featureFlags() + .allowMoreThanOneContentGroupDown(new ClusterSpec.Id(c.clusterId))) .build(deployState, c, contentElement.getXml()); - config.tuning().maxGroupsAllowedDown().ifPresent(m -> { - int numberOfLeafGroups = c.getRootGroup().getNumberOfLeafGroups(); - if (m > numberOfLeafGroups) - throw new IllegalArgumentException("Cannot set max-groups-allowed-down (" + m + ") larger than number of groups (" + numberOfLeafGroups + ")"); - }); - return config; } private void setupSearchCluster(ContentSearchCluster csc, ModelElement element, DeployLogger logger) { diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 9d6e16b3f67..9a38fdc673e 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -1075,15 +1075,16 @@ void attributeSetting(ParsedAttribute attribute) : void summaryInDocument(ParsedDocumentSummary docsum) : { String name; - ParsedType type; + ParsedType type = null; ParsedSummaryField psf; } { <SUMMARY> name = identifierWithDash() { } - <TYPE> type = dataType() { + (<TYPE> type = dataType())? + lbrace() { psf = new ParsedSummaryField(name, type); } - lbrace() (summaryItem(psf) (<NL>)*)* <RBRACE> + (summaryItem(psf) (<NL>)*)* <RBRACE> { var old = docsum.addField(psf); if (old != null) { diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 392572e1f12..98ab2e61783 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -112,7 +112,8 @@ LogForwarding = element logforwarding { attribute splunk-home { xsd:string }? & attribute deployment-server { xsd:string } & attribute client-name { xsd:string } & - attribute phone-home-interval { xsd:positiveInteger }? + attribute phone-home-interval { xsd:positiveInteger }? & + attribute role { xsd:string }? } } diff --git a/config-model/src/test/derived/multiplesummaries/index-info.cfg b/config-model/src/test/derived/multiplesummaries/index-info.cfg index 085a9eb232f..50f2419fc58 100644 --- a/config-model/src/test/derived/multiplesummaries/index-info.cfg +++ b/config-model/src/test/derived/multiplesummaries/index-info.cfg @@ -82,8 +82,6 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "h" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "h" -indexinfo[].command[].command "string" -indexinfo[].command[].indexname "h" indexinfo[].command[].command "type WeightedSet<string>" indexinfo[].command[].indexname "loc" indexinfo[].command[].command "index" diff --git a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd index 51259802a3a..5f93a6e512b 100644 --- a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd +++ b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd @@ -62,6 +62,7 @@ schema multiplesummaries { field h type weightedset<string> { indexing: summary + weightedset: create-if-nonexistent } field loc type string { @@ -91,7 +92,7 @@ schema multiplesummaries { summary e type string { } - summary f type array<string> { + summary f { } summary g type array<int> { @@ -209,7 +210,7 @@ schema multiplesummaries { bolding: on } - summary c type string { + summary c { } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 90b4625a282..f1dffe53ad7 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -15,6 +15,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorStatusConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; @@ -2352,6 +2353,38 @@ public class ModelProvisioningTest { } @Test + public void testAllow2ContentGroupsDown() { + String servicesXml = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='qrs'>" + + " <nodes count='1'/>" + + " </container>" + + " <content version='1.0' id='content'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='4' groups='4'/>" + + " <tuning>" + + " <cluster-controller>" + + " <max-groups-allowed-down>2</max-groups-allowed-down>" + + " </cluster-controller>" + + " </tuning>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.setModelProperties(new TestProperties().setAllowMoreThanOneContentGroupDown(true)); + tester.addHosts(9); + VespaModel model = tester.createModel(servicesXml, true, new DeployState.Builder() + .properties(new TestProperties().setAllowMoreThanOneContentGroupDown(true))); + + var fleetControllerConfigBuilder = new FleetcontrollerConfig.Builder(); + model.getConfig(fleetControllerConfigBuilder, "admin/standalone/cluster-controllers/0/components/clustercontroller-content-configurer"); + assertEquals(2, fleetControllerConfigBuilder.build().max_number_of_groups_allowed_to_be_down()); + } + + @Test public void containerWithZooKeeperSuboptimalNodeCountDuringRetirement() { String servicesXml = "<?xml version='1.0' encoding='utf-8' ?>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index a8ffc625ee6..b809f25ced2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -146,7 +146,7 @@ public class DedicatedAdminV4Test { " <slobroks><nodes count='2' dedicated='true'/></slobroks>" + " <logservers><nodes count='1' dedicated='true'/></logservers>" + " <logforwarding include-admin='true'>" + - " <splunk deployment-server='foo:123' client-name='foocli' phone-home-interval='900'/>" + + " <splunk deployment-server='foo:123' client-name='foocli' phone-home-interval='900' role='athenz://some-domain/role/role-name'/>" + " </logforwarding>" + " </admin>" + "</services>"; @@ -176,6 +176,7 @@ public class DedicatedAdminV4Test { assertEquals("foocli", config.clientName()); assertEquals("/opt/splunkforwarder", config.splunkHome()); assertEquals(900, config.phoneHomeInterval()); + assertEquals("some-domain:role.role-name", config.role()); } // Other host's forwarder @@ -188,6 +189,7 @@ public class DedicatedAdminV4Test { assertEquals("foocli", config.clientName()); assertEquals("/opt/splunkforwarder", config.splunkHome()); assertEquals(900, config.phoneHomeInterval()); + assertEquals("some-domain:role.role-name", config.role()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java index 3feb8888821..1c22423147c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java @@ -21,7 +21,7 @@ public class ContainerInCloudValidatorTest { void failsWhenNoContainerInCloud() throws IOException, SAXException { String noContainer = ""; String container = """ - <container id='default' version='1.0'> + <container id='routing' version='1.0'> <nodes count='2' /> </container> """; @@ -38,7 +38,7 @@ public class ContainerInCloudValidatorTest { String servicesXml = """ <services version='1.0'> %s - <content version='1.0'> + <content id='foo' version='1.0'> <redundancy>2</redundancy> <documents> </documents> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index a33b30f7d93..12a6ac00f48 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1331,14 +1331,10 @@ public class ContentClusterTest extends ContentBaseTest { " </engine>" + " </content>" + " </services>"; - VespaModel model = createEnd2EndOneNode(new TestProperties() - .setHostedVespa(false) - .setMultitenant(true) - .setAllowMoreThanOneContentGroupDown(true), - services); + VespaModel model = createEnd2EndOneNode(new TestProperties().setAllowMoreThanOneContentGroupDown(true), services); var fleetControllerConfigBuilder = new FleetcontrollerConfig.Builder(); - model.getConfig(fleetControllerConfigBuilder, "admin/standalone/cluster-controllers/0/components/clustercontroller-storage-configurer"); + model.getConfig(fleetControllerConfigBuilder, "admin/cluster-controllers/0/components/clustercontroller-storage-configurer"); assertEquals(2, fleetControllerConfigBuilder.build().max_number_of_groups_allowed_to_be_down()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index ae22542de6c..138852e1c5c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -41,14 +41,14 @@ public class FleetControllerClusterTest { parse(""" <cluster id="storage"> <documents/> <tuning> - <bucket-splitting minimum-bits="7" /> <cluster-controller> + <bucket-splitting minimum-bits="7" /> + <cluster-controller> <init-progress-time>13</init-progress-time> <transition-time>27</transition-time> <max-premature-crashes>4</max-premature-crashes> <stable-state-period>72</stable-state-period> <min-distributor-up-ratio>0.7</min-distributor-up-ratio> <min-storage-up-ratio>0.3</min-storage-up-ratio> - <max-groups-allowed-down>2</max-groups-allowed-down> </cluster-controller> </tuning> </cluster>""", @@ -63,7 +63,6 @@ public class FleetControllerClusterTest { assertEquals(0.7, config.min_distributor_up_ratio(), 0.01); assertEquals(0.3, config.min_storage_up_ratio(), 0.01); assertEquals(7, config.ideal_distribution_bits()); - assertEquals(2, config.max_number_of_groups_allowed_to_be_down()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 48ddf6b8a82..500fb0838e1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -21,7 +21,6 @@ import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; - import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -50,6 +49,7 @@ public class VespaModelTester { private final ConfigModelRegistry configModelRegistry; private boolean hosted = true; + private TestProperties modelProperties = new TestProperties(); private final Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>(); private ApplicationId applicationId = ApplicationId.defaultId(); private boolean useDedicatedNodeForLogserver = false; @@ -101,6 +101,9 @@ public class VespaModelTester { /** Sets whether this sets up a model for a hosted system. Default: true */ public void setHosted(boolean hosted) { this.hosted = hosted; } + /** Sets whether this sets up a model for a hosted system. Default: true */ + public void setModelProperties(TestProperties testProperties) { this.modelProperties = testProperties; } + /** Sets architecture to use for admin clusters. Default: x86_64 */ public void setAdminClusterArchitecture(Architecture architecture) { this.adminClusterArchitecture = architecture; @@ -206,7 +209,7 @@ public class VespaModelTester { provisioner = new SingleNodeProvisioner(); } - TestProperties properties = new TestProperties() + TestProperties properties = modelProperties .setMultitenant(hosted) // Note: system tests are multitenant but not hosted .setHostedVespa(hosted) .setApplicationId(applicationId) diff --git a/configdefinitions/src/vespa/logforwarder.def b/configdefinitions/src/vespa/logforwarder.def index 60a607098e0..4f6b3fc61a7 100644 --- a/configdefinitions/src/vespa/logforwarder.def +++ b/configdefinitions/src/vespa/logforwarder.def @@ -7,3 +7,4 @@ deploymentServer string default="" clientName string default="" splunkHome string default="/opt/splunkforwarder" phoneHomeInterval int default=60 +role string default="" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index b9118602058..da18c4e4fcc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -175,25 +175,20 @@ public class FileDirectory extends AbstractComponent { ensureRootExist(); Path tempDestinationDir = uncheck(() -> Files.createTempDirectory(root.toPath(), "writing")); try { - // Prepare and verify logfileInfo(source); - File destinationDir = destinationDir(reference); - File tempDestination = new File(tempDestinationDir.toFile(), source.getName()); - if ( ! destinationDir.mkdir()) - log.log(Level.WARNING, () -> "destination dir " + destinationDir + " already exists"); - // Copy files + // Copy files to temp dir + File tempDestination = new File(tempDestinationDir.toFile(), source.getName()); log.log(Level.FINE, () -> "Copying " + source.getAbsolutePath() + " to " + tempDestination.getAbsolutePath()); if (source.isDirectory()) IOUtils.copyDirectory(source, tempDestination, -1); else copyFile(source, tempDestination); - // Move to final destination - log.log(Level.FINE, () -> "Moving " + tempDestinationDir + " to " + destinationDir.getAbsolutePath()); - if ( ! tempDestinationDir.toFile().renameTo(destinationDir)) - log.log(Level.WARNING, "Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + - "' to '" + tempDestination.getAbsolutePath() + "'."); + // Move to destination dir + Path destinationDir = destinationDir(reference).toPath(); + log.log(Level.FINE, () -> "Moving " + tempDestinationDir + " to " + destinationDir); + Files.move(tempDestinationDir, destinationDir); return reference; } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index 70dff6730ff..409b807c833 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -9,9 +9,7 @@ import com.yahoo.documentapi.VisitorDataHandler; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; -import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.messagebus.Message; @@ -300,12 +298,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { public void onMessage(Message m, AckToken token) { if (m instanceof QueryResultMessage qm) { onQueryResult(qm.getResult(), qm.getSummary()); - } else if (m instanceof SearchResultMessage) { - onSearchResult(((SearchResultMessage) m).getResult()); - } else if (m instanceof DocumentSummaryMessage dsm) { - onDocumentSummary(dsm.getResult()); } else { - throw new UnsupportedOperationException("Received unsupported message " + m + ". VdsVisitor can only accept query result, search result, and documentsummary messages."); + throw new UnsupportedOperationException("Received unsupported message " + m + ". VdsVisitor can only accept query result messages."); } ack(token); } @@ -320,13 +314,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { handleSummary(summary); } - public void onSearchResult(SearchResult sr) { - if (log.isLoggable(Level.FINEST)) { - log.log(Level.FINEST, "Got SearchResult for query with selection " + params.getDocumentSelection()); - } - handleSearchResult(sr); - } - private void handleSearchResult(SearchResult sr) { final int hitCountTotal = sr.getTotalHitCount(); final int hitCount = sr.getHitCount(); @@ -377,13 +364,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } } - public void onDocumentSummary(DocumentSummary ds) { - if (log.isLoggable(Level.FINEST)) { - log.log(Level.FINEST, "Got DocumentSummary for query with selection " + params.getDocumentSelection()); - } - handleSummary(ds); - } - private void handleSummary(DocumentSummary ds) { int summaryCount = ds.getSummaryCount(); if (log.isLoggable(Level.FINE)) { diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java index 0e3ecf1c8cc..28a34ff2f6d 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java @@ -5,9 +5,7 @@ import com.yahoo.document.fieldset.AllFields; import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.*; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; -import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.routing.Route; @@ -63,18 +61,6 @@ public class VdsVisitorTestCase { return qrm; } - private SearchResultMessage createSRM(String docId, double rank) { - SearchResultMessage srm = new SearchResultMessage(); - srm.setSearchResult(createSR(docId, rank)); - return srm; - } - - private DocumentSummaryMessage createDSM(String docId) { - DocumentSummaryMessage dsm = new DocumentSummaryMessage(); - dsm.setDocumentSummary(createDS(docId)); - return dsm; - } - private Message createM() { return new Message() { @Override @@ -357,15 +343,13 @@ public class VdsVisitorTestCase { private void supplyResults(VdsVisitor visitor) { AckToken ackToken = null; visitor.onMessage(createQRM("id:ns:type::0", 0.3), ackToken); - visitor.onMessage(createSRM("id:ns:type::1", 1.0), ackToken); - visitor.onMessage(createSRM("id:ns:type::2", 0.5), ackToken); - visitor.onMessage(createDSM("id:ns:type::1"), ackToken); - visitor.onMessage(createDSM("id:ns:type::2"), ackToken); + visitor.onMessage(createQRM("id:ns:type::1", 1.0), ackToken); + visitor.onMessage(createQRM("id:ns:type::2", 0.5), ackToken); try { visitor.onMessage(createM(), ackToken); assertTrue(false, "Unsupported message did not cause exception"); } catch (UnsupportedOperationException uoe) { - assertTrue(uoe.getMessage().contains("VdsVisitor can only accept query result, search result, and documentsummary messages")); + assertTrue(uoe.getMessage().contains("VdsVisitor can only accept query result messages")); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java index 796ce5da449..65c8e8390c8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import java.util.Objects; @@ -22,15 +23,20 @@ public class NodeFilter { private final Set<HostName> hostnames; private final Set<HostName> parentHostnames; private final Set<ApplicationId> applications; + private final Set<ClusterSpec.Id> clusterIds; + private final Set<Node.ClusterType> clusterTypes; private NodeFilter(boolean includeDeprovisioned, Set<Node.State> states, Set<HostName> hostnames, - Set<HostName> parentHostnames, Set<ApplicationId> applications) { + Set<HostName> parentHostnames, Set<ApplicationId> applications, + Set<ClusterSpec.Id> clusterIds, Set<Node.ClusterType> clusterTypes) { this.includeDeprovisioned = includeDeprovisioned; // Uses Guava Set to preserve insertion order this.states = ImmutableSet.copyOf(Objects.requireNonNull(states)); this.hostnames = ImmutableSet.copyOf(Objects.requireNonNull(hostnames)); this.parentHostnames = ImmutableSet.copyOf(Objects.requireNonNull(parentHostnames)); this.applications = ImmutableSet.copyOf(Objects.requireNonNull(applications)); + this.clusterIds = ImmutableSet.copyOf(Objects.requireNonNull(clusterIds)); + this.clusterTypes = ImmutableSet.copyOf(Objects.requireNonNull(clusterTypes)); if (!includeDeprovisioned && states.contains(Node.State.deprovisioned)) { throw new IllegalArgumentException("Must include deprovisioned nodes when matching deprovisioned state"); } @@ -56,8 +62,16 @@ public class NodeFilter { return applications; } + public Set<ClusterSpec.Id> clusterIds() { + return clusterIds; + } + + public Set<Node.ClusterType> clusterTypes() { + return clusterTypes; + } + public NodeFilter includeDeprovisioned(boolean includeDeprovisioned) { - return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); } public NodeFilter states(Node.State... states) { @@ -65,7 +79,7 @@ public class NodeFilter { } public NodeFilter states(Set<Node.State> states) { - return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); } public NodeFilter hostnames(HostName... hostnames) { @@ -73,7 +87,7 @@ public class NodeFilter { } public NodeFilter hostnames(Set<HostName> hostnames) { - return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); } public NodeFilter parentHostnames(HostName... parentHostnames) { @@ -81,7 +95,7 @@ public class NodeFilter { } public NodeFilter parentHostnames(Set<HostName> parentHostnames) { - return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); } public NodeFilter applications(ApplicationId... applications) { @@ -89,12 +103,28 @@ public class NodeFilter { } public NodeFilter applications(Set<ApplicationId> applications) { - return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); + } + + public NodeFilter clusterIds(ClusterSpec.Id... clusterIds) { + return clusterIds(ImmutableSet.copyOf(clusterIds)); + } + + public NodeFilter clusterIds(Set<ClusterSpec.Id> clusterIds) { + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); + } + + public NodeFilter clusterTypes(Node.ClusterType... clusterTypes) { + return clusterTypes(ImmutableSet.copyOf(clusterTypes)); + } + + public NodeFilter clusterTypes(Set<Node.ClusterType> clusterTypes) { + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications, clusterIds, clusterTypes); } /** A filter which matches all nodes, except deprovisioned ones */ public static NodeFilter all() { - return new NodeFilter(false, Set.of(), Set.of(), Set.of(), Set.of()); + return new NodeFilter(false, Set.of(), Set.of(), Set.of(), Set.of(), Set.of(), Set.of()); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 4c5a67626ea..485bf627c87 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; @@ -11,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.Applicat import java.util.List; import java.util.Map; +import java.util.Optional; /** * Node repository interface intended for use by the controller. @@ -67,6 +69,9 @@ public interface NodeRepository { /** Retire given node */ void retire(ZoneId zone, String hostname, boolean wantToRetire, boolean wantToDeprovision); + /** Drop all documents on content nodes in the given zone, application and cluster */ + void dropDocuments(ZoneId zoneId, ApplicationId applicationId, Optional<ClusterSpec.Id> clusterId); + /** Update reports for given node. A key with null value clears that report */ void updateReports(ZoneId zone, String hostname, Map<String, String> reports); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index fef29a99a47..ac895022130 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -173,6 +173,10 @@ enum PathGroup { Matcher.application, "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/node/{node}/service-dump"), + dropDocuments(Matcher.tenant, + Matcher.application, + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/drop-documents"), + /** Paths used for development deployments. */ developmentDeployment(Matcher.tenant, Matcher.application, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index bb53ae61525..9a28226c921 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -87,7 +87,7 @@ enum Policy { /** Read access to application information and settings. */ applicationRead(Privilege.grant(Action.read) - .on(PathGroup.application, PathGroup.applicationInfo, PathGroup.reindexing, PathGroup.serviceDump) + .on(PathGroup.application, PathGroup.applicationInfo, PathGroup.reindexing, PathGroup.serviceDump, PathGroup.dropDocuments) .in(SystemName.all())), /** Update access to application information and settings. */ @@ -102,7 +102,7 @@ enum Policy { /** Full access to application information and settings. */ applicationOperations(Privilege.grant(Action.write()) - .on(PathGroup.applicationInfo, PathGroup.productionRestart, PathGroup.reindexing, PathGroup.serviceDump) + .on(PathGroup.applicationInfo, PathGroup.productionRestart, PathGroup.reindexing, PathGroup.serviceDump, PathGroup.dropDocuments) .in(SystemName.all())), /** Access to create and delete developer and deploy keys under a tenant. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 4a8bc3cd09a..3ebaebf680a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -1,15 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application.pkg; -import com.google.common.hash.Funnel; -import com.google.common.hash.HashFunction; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; import com.google.common.hash.HashingOutputStream; import com.yahoo.component.Version; -import com.yahoo.vespa.archive.ArchiveStreamReader; -import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; -import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import com.yahoo.config.application.FileSystemWrapper; import com.yahoo.config.application.FileSystemWrapper.FileWrapper; import com.yahoo.config.application.XmlPreProcessor; @@ -23,10 +18,12 @@ import com.yahoo.config.provision.Tags; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.archive.ArchiveStreamReader; +import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; +import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder; import com.yahoo.yolean.Exceptions; - import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; @@ -44,10 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Random; import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.function.Function; import java.util.function.Predicate; @@ -66,7 +60,8 @@ import static java.util.stream.Collectors.toMap; */ public class ApplicationPackage { - static final String trustedCertificatesFile = "security/clients.pem"; + static final String trustedCertificatesDir = "security/"; + static final String trustedCertificatesFile = trustedCertificatesDir + "clients.pem"; static final String buildMetaFile = "build-meta.json"; static final String deploymentFile = "deployment.xml"; static final String validationOverridesFile = "validation-overrides.xml"; @@ -90,7 +85,7 @@ public class ApplicationPackage { * it must not be further changed by the caller. */ public ApplicationPackage(byte[] zippedContent) { - this(zippedContent, false); + this(zippedContent, false, false); } /** @@ -99,9 +94,9 @@ public class ApplicationPackage { * it must not be further changed by the caller. * If 'requireFiles' is true, files needed by deployment orchestration must be present. */ - public ApplicationPackage(byte[] zippedContent, boolean requireFiles) { + public ApplicationPackage(byte[] zippedContent, boolean requireFiles, boolean checkCertificateFile) { this.zippedContent = Objects.requireNonNull(zippedContent, "The application package content cannot be null"); - this.files = new ZipArchiveCache(zippedContent, prePopulated); + this.files = new ZipArchiveCache(zippedContent, prePopulated, checkCertificateFile); Optional<DeploymentSpec> deploymentSpec = files.get(deploymentFile).map(bytes -> new String(bytes, UTF_8)).map(DeploymentSpec::fromXml); if (requireFiles && deploymentSpec.isEmpty()) @@ -253,10 +248,12 @@ public class ApplicationPackage { private final byte[] zip; private final Map<Path, Optional<byte[]>> cache; - public ZipArchiveCache(byte[] zip, Collection<String> prePopulated) { + public ZipArchiveCache(byte[] zip, Collection<String> prePopulated, boolean checkCertificateFile) { this.zip = zip; this.cache = new ConcurrentSkipListMap<>(); this.cache.putAll(read(prePopulated)); + if (checkCertificateFile) + verifyThatTrustedCertificateExists(); } public Optional<byte[]> get(String path) { @@ -274,17 +271,26 @@ public class ApplicationPackage { } private Map<Path, Optional<byte[]>> read(Collection<String> names) { - var entries = ZipEntries.from(zip, - names::contains, - maxSize, - true) - .asList().stream() - .collect(toMap(entry -> Paths.get(entry.name()).normalize(), - ZipEntries.ZipEntryWithContent::content)); + var entries = findZipFileEntries(names::contains); names.stream().map(Paths::get).forEach(path -> entries.putIfAbsent(path.normalize(), Optional.empty())); return entries; } + + private void verifyThatTrustedCertificateExists() { + // Any name is valid for certificate files + var entries = findZipFileEntries((entry) -> entry.contains(trustedCertificatesDir) && entry.endsWith(".pem")); + if (entries.size() == 0) + throw new IllegalArgumentException("No client certificate found in " + trustedCertificatesDir + " in application package" + + ", see https://cloud.vespa.ai/en/security/guide"); + } + + private Map<Path, Optional<byte[]>> findZipFileEntries(Predicate<String> names) { + return ZipEntries.from(zip, names, maxSize, true) + .asList().stream() + .collect(toMap(entry -> Paths.get(entry.name()).normalize(), + ZipEntries.ZipEntryWithContent::content)); + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index ded27ee1060..9224c53136d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -52,6 +52,8 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.text.Text; import com.yahoo.vespa.athenz.api.OAuthCredentials; import com.yahoo.vespa.athenz.client.zms.ZmsClientException; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -165,6 +167,7 @@ import java.util.stream.Stream; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.CONFLICT; +import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; import static com.yahoo.yolean.Exceptions.uncheck; import static java.util.Comparator.comparingInt; import static java.util.Map.Entry.comparingByKey; @@ -186,6 +189,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private final Controller controller; private final AccessControlRequests accessControlRequests; private final TestConfigSerializer testConfigSerializer; + private final BooleanFlag failDeploymentOnMissingCertificateFile; @Inject public ApplicationApiHandler(ThreadedHttpRequestHandler.Context parentCtx, @@ -195,6 +199,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { this.controller = controller; this.accessControlRequests = accessControlRequests; this.testConfigSerializer = new TestConfigSerializer(controller.system()); + this.failDeploymentOnMissingCertificateFile = Flags.FAIL_DEPLOYMENT_ON_MISSING_CERTIFICATE_FILE.bindTo(controller.flagSource()); } @Override @@ -281,6 +286,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/content/{*}")) return content(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/private-services")) return getPrivateServiceInfo(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/drop-documents")) return dropDocumentsStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return supportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/node/{node}/service-dump")) return getServiceDump(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("node"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/scaling")) return scaling(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -346,6 +352,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return enableReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspend")) return suspend(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/drop-documents")) return dropDocuments(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/access/support")) return allowSupportAccess(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/node/{node}/service-dump")) return requestServiceDump(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("node"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploySystemApplication(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -2017,6 +2024,65 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(slime); } + private HttpResponse dropDocumentsStatus(String tenant, String application, String instance, String environment, String region, Optional<ClusterSpec.Id> clusterId) { + ZoneId zone = ZoneId.from(environment, region); + if (!zone.environment().isManuallyDeployed()) + throw new IllegalArgumentException("Drop documents status is only available for manually deployed environments"); + + ApplicationId applicationId = ApplicationId.from(tenant, application, instance); + NodeFilter filters = NodeFilter.all() + .states(Node.State.active) + .applications(applicationId) + .clusterTypes(Node.ClusterType.content, Node.ClusterType.combined); + List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, clusterId.map(filters::clusterIds).orElse(filters)); + if (nodes.isEmpty()) { + throw new NotExistsException("No content nodes found for %s%s in %s".formatted( + applicationId.toFullString(), clusterId.map(id -> " cluster " + id).orElse(""), zone)); + } + + Instant readiedAt = null; + int numNoReport = 0, numInitial = 0, numDropped = 0, numReadied = 0, numStarted = 0; + for (Node node : nodes) { + Inspector report = Optional.ofNullable(node.reports().get("dropDocuments")) + .map(json -> SlimeUtils.jsonToSlime(json).get()).orElse(null); + if (report == null) numNoReport++; + else if (report.field("startedAt").valid()) { + numStarted++; + readiedAt = SlimeUtils.instant(report.field("readiedAt")); + } else if (report.field("readiedAt").valid()) numReadied++; + else if (report.field("droppedAt").valid()) numDropped++; + else numInitial++; + } + + if (numInitial + numDropped > 0 && numNoReport + numReadied + numStarted > 0) + return ErrorResponse.conflict("Last dropping of documents may have failed to clear all documents due " + + "to concurrent topology changes, consider retrying"); + + Slime slime = new Slime(); + Cursor root = slime.setObject(); + if (numStarted + numNoReport == nodes.size()) { + if (readiedAt != null) root.setLong("lastDropped", readiedAt.toEpochMilli()); + } else { + Cursor progress = root.setObject("progress"); + progress.setLong("total", nodes.size()); + progress.setLong("dropped", numDropped + numReadied + numStarted); + progress.setLong("started", numStarted + numNoReport); + } + + return new SlimeJsonResponse(slime); + } + + private HttpResponse dropDocuments(String tenant, String application, String instance, String environment, String region, Optional<ClusterSpec.Id> clusterId) { + ZoneId zone = ZoneId.from(environment, region); + if (!zone.environment().isManuallyDeployed()) + throw new IllegalArgumentException("Drop documents status is only available for manually deployed environments"); + + ApplicationId applicationId = ApplicationId.from(tenant, application, instance); + controller.serviceRegistry().configServer().nodeRepository().dropDocuments(zone, applicationId, clusterId); + return new MessageResponse("Triggered drop documents for " + applicationId.toFullString() + + clusterId.map(id -> " and cluster " + id).orElse("") + " in " + zone); + } + private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); @@ -3004,7 +3070,12 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { throw new IllegalArgumentException("Source URL must include scheme and host"); }); - ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP), true); + ApplicationPackage applicationPackage = + new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP), + true, + failDeploymentOnMissingCertificateFile + .with(APPLICATION_ID, ApplicationId.from(tenant, application, "default").serializedForm()) + .value()); byte[] testPackage = dataParts.getOrDefault(EnvironmentResource.APPLICATION_TEST_ZIP, new byte[0]); Submission submission = new Submission(applicationPackage, testPackage, sourceUrl, sourceRevision, authorEmail, description, risk); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index 7f578d3017e..e915a204e4b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.io.LazyInputStream; import org.junit.jupiter.api.Test; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -97,7 +96,7 @@ public class ApplicationPackageTest { "jdisc.xml", jdiscXml, "content/content.xml", contentXml, "content/nodes.xml", nodesXml), - unzip(new ApplicationPackage(zip, false).metaDataZip())); + unzip(new ApplicationPackage(zip).metaDataZip())); } @Test @@ -105,7 +104,7 @@ public class ApplicationPackageTest { byte[] zip = filesZip(Map.of("services.xml", servicesXml.getBytes(UTF_8))); try { - new ApplicationPackage(zip, false).metaDataZip(); + new ApplicationPackage(zip).metaDataZip(); fail("Should fail on missing include file"); } catch (RuntimeException e) { @@ -152,6 +151,21 @@ public class ApplicationPackageTest { assertEquals(originalPackage.bundleHash(), similarDeploymentXml.bundleHash()); } + @Test + void testCertificateFileExists() throws Exception { + getApplicationZip("with-certificate.zip", true); + } + + @Test + void testCertificateFileMissing() throws Exception { + try { + getApplicationZip("original.zip", true); + fail("Should fail on missing certificate file file"); + } catch (RuntimeException e) { + assertEquals("No client certificate found in security/ in application package, see https://cloud.vespa.ai/en/security/guide", e.getMessage()); + } + } + static Map<String, String> unzip(byte[] zip) { return ZipEntries.from(zip, __ -> true, 1 << 24, true) .asList().stream() @@ -160,7 +174,11 @@ public class ApplicationPackageTest { } private ApplicationPackage getApplicationZip(String path) throws IOException { - return new ApplicationPackage(Files.readAllBytes(Path.of("src/test/resources/application-packages/" + path)), true); + return getApplicationZip(path, false); + } + + private ApplicationPackage getApplicationZip(String path, boolean checkCertificateFile) throws IOException { + return new ApplicationPackage(Files.readAllBytes(Path.of("src/test/resources/application-packages/" + path)), true, checkCertificateFile); } static byte[] zip(Map<String, String> content) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 297997365b0..7004028c072 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -88,6 +88,8 @@ public class NodeRepositoryMock implements NodeRepository { (node.owner().isPresent() && filter.applications().contains(node.owner().get()))) .filter(node -> filter.hostnames().isEmpty() || filter.hostnames().contains(node.hostname())) .filter(node -> filter.states().isEmpty() || filter.states().contains(node.state())) + .filter(node -> filter.clusterIds().isEmpty() || filter.clusterIds().contains(ClusterSpec.Id.from(node.clusterId()))) + .filter(node -> filter.clusterTypes().isEmpty() || filter.clusterTypes().contains(node.clusterType())) .toList(); } @@ -201,6 +203,10 @@ public class NodeRepositoryMock implements NodeRepository { } @Override + public void dropDocuments(ZoneId zoneId, ApplicationId applicationId, Optional<ClusterSpec.Id> clusterId) { + } + + @Override public void updateReports(ZoneId zone, String hostname, Map<String, String> reports) { Map<String, String> trimmedReports = reports.entrySet().stream() // Null value clears a report diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 76bcbe078ff..c6d68bc5d9d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -40,6 +41,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; @@ -55,6 +57,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.notification.Notification; @@ -90,6 +93,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.BiFunction; import java.util.function.Supplier; import static com.yahoo.application.container.handler.Request.Method.DELETE; @@ -510,6 +514,42 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/content/bar/file.json?query=param", GET).userIdentity(USER_ID), "{\"path\":\"/bar/file.json\"}"); + // Drop documents + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/drop-documents", POST) + .userIdentity(USER_ID), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Drop documents status is only available for manually deployed environments\"}", 400); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", POST) + .userIdentity(USER_ID), + "{\"message\":\"Triggered drop documents for tenant2.application1.default in dev.us-east-1\"}"); + + ZoneId zone = ZoneId.from("dev", "us-east-1"); + ApplicationId application = ApplicationId.from("tenant2", "application1", "default"); + BiFunction<Integer, String, Node> nodeBuilder = (index, dropDocumentsReport) -> Node.builder().hostname("node" + index + ".dev.us-east-1.test") + .state(Node.State.active).type(NodeType.tenant).owner(application).clusterId("c1").clusterType(Node.ClusterType.content) + .reports(dropDocumentsReport == null ? Map.of() : Map.of("dropDocuments", dropDocumentsReport)).build(); + NodeRepositoryMock nodeRepository = deploymentTester.controllerTester().serviceRegistry().configServer().nodeRepository(); + + // 2 nodes, neither ever dropped any documents + nodeRepository.putNodes(zone, List.of(nodeBuilder.apply(1, null), nodeBuilder.apply(2, null))); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", GET).userIdentity(USER_ID), + "{}"); + + // 1 node previously dropped documents, 1 node without any report + nodeRepository.putNodes(zone, List.of(nodeBuilder.apply(1, "{\"droppedAt\":1,\"readiedAt\":2,\"startedAt\":3}"), nodeBuilder.apply(2, null))); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", GET).userIdentity(USER_ID), + "{\"lastDropped\":2}"); + + nodeRepository.putNodes(zone, List.of(nodeBuilder.apply(1, "{}"), nodeBuilder.apply(2, null))); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", GET).userIdentity(USER_ID), + "{\"error-code\":\"CONFLICT\",\"message\":\"Last dropping of documents may have failed to clear all documents due to concurrent topology changes, consider retrying\"}", 409); + + nodeRepository.putNodes(zone, List.of(nodeBuilder.apply(1, "{}"), nodeBuilder.apply(2, "{\"droppedAt\":1}"))); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", GET).userIdentity(USER_ID), + "{\"progress\":{\"total\":2,\"dropped\":1,\"started\":0}}"); + + nodeRepository.putNodes(zone, List.of(nodeBuilder.apply(1, "{\"startedAt\":3}"), nodeBuilder.apply(2, "{\"readiedAt\":1}"))); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/drop-documents", GET).userIdentity(USER_ID), + "{\"progress\":{\"total\":2,\"dropped\":2,\"started\":1}}"); updateMetrics(); diff --git a/controller-server/src/test/resources/application-packages/with-certificate.zip b/controller-server/src/test/resources/application-packages/with-certificate.zip Binary files differnew file mode 100644 index 00000000000..1540b96c7ef --- /dev/null +++ b/controller-server/src/test/resources/application-packages/with-certificate.zip diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index dbea284e39e..73ad4b1d121 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -1828,8 +1828,6 @@ "public static final int MESSAGE_CREATEVISITOR", "public static final int MESSAGE_DESTROYVISITOR", "public static final int MESSAGE_VISITORINFO", - "public static final int MESSAGE_SEARCHRESULT", - "public static final int MESSAGE_DOCUMENTSUMMARY", "public static final int MESSAGE_MAPVISITOR", "public static final int MESSAGE_GETBUCKETSTATE", "public static final int MESSAGE_STATBUCKET", @@ -1846,8 +1844,6 @@ "public static final int REPLY_CREATEVISITOR", "public static final int REPLY_DESTROYVISITOR", "public static final int REPLY_VISITORINFO", - "public static final int REPLY_SEARCHRESULT", - "public static final int REPLY_DOCUMENTSUMMARY", "public static final int REPLY_MAPVISITOR", "public static final int REPLY_GETBUCKETSTATE", "public static final int REPLY_STATBUCKET", @@ -2093,21 +2089,6 @@ ], "fields" : [ ] }, - "com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.VisitorMessage", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "public void setDocumentSummary(com.yahoo.vdslib.DocumentSummary)", - "public com.yahoo.vdslib.DocumentSummary getResult()", - "public com.yahoo.documentapi.messagebus.protocol.DocumentReply createReply()", - "public int getType()" - ], - "fields" : [ ] - }, "com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig$Builder" : { "superClass" : "java.lang.Object", "interfaces" : [ @@ -2459,8 +2440,8 @@ "public int getType()", "public com.yahoo.document.TestAndSetCondition getCondition()", "public void setCondition(com.yahoo.document.TestAndSetCondition)", - "public boolean getCreateIfNonExistent()", - "public void setCreateIfNonExistent(boolean)" + "public void setCreateIfNonExistent(boolean)", + "public boolean getCreateIfNonExistent()" ], "fields" : [ ] }, @@ -2672,32 +2653,6 @@ ], "fields" : [ ] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentSummaryMessageFactory" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "protected com.yahoo.documentapi.messagebus.protocol.DocumentMessage doDecode(com.yahoo.document.serialization.DocumentDeserializer)", - "protected boolean doEncode(com.yahoo.documentapi.messagebus.protocol.DocumentMessage, com.yahoo.document.serialization.DocumentSerializer)" - ], - "fields" : [ ] - }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentSummaryReplyFactory" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "protected com.yahoo.documentapi.messagebus.protocol.DocumentReply doDecode(com.yahoo.document.serialization.DocumentDeserializer)", - "protected boolean doEncode(com.yahoo.documentapi.messagebus.protocol.DocumentReply, com.yahoo.document.serialization.DocumentSerializer)" - ], - "fields" : [ ] - }, "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$EmptyBucketsMessageFactory" : { "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces" : [ ], @@ -2936,32 +2891,6 @@ ], "fields" : [ ] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$SearchResultMessageFactory" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "protected com.yahoo.documentapi.messagebus.protocol.DocumentMessage doDecode(com.yahoo.document.serialization.DocumentDeserializer)", - "protected boolean doEncode(com.yahoo.documentapi.messagebus.protocol.DocumentMessage, com.yahoo.document.serialization.DocumentSerializer)" - ], - "fields" : [ ] - }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$SearchResultReplyFactory" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "protected com.yahoo.documentapi.messagebus.protocol.DocumentReply doDecode(com.yahoo.document.serialization.DocumentDeserializer)", - "protected boolean doEncode(com.yahoo.documentapi.messagebus.protocol.DocumentReply, com.yahoo.document.serialization.DocumentSerializer)" - ], - "fields" : [ ] - }, "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$StatBucketMessageFactory" : { "superClass" : "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces" : [ ], @@ -3095,21 +3024,6 @@ ], "fields" : [ ] }, - "com.yahoo.documentapi.messagebus.protocol.SearchResultMessage" : { - "superClass" : "com.yahoo.documentapi.messagebus.protocol.VisitorMessage", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>()", - "public com.yahoo.vdslib.SearchResult getResult()", - "public void setSearchResult(com.yahoo.vdslib.SearchResult)", - "public com.yahoo.documentapi.messagebus.protocol.DocumentReply createReply()", - "public int getType()" - ], - "fields" : [ ] - }, "com.yahoo.documentapi.messagebus.protocol.SlobrokPolicy" : { "superClass" : "java.lang.Object", "interfaces" : [ diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 0ff578b64d7..6185437a48f 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -50,8 +50,9 @@ public class DocumentProtocol implements Protocol { public static final int MESSAGE_CREATEVISITOR = DOCUMENT_MESSAGE + 7; public static final int MESSAGE_DESTROYVISITOR = DOCUMENT_MESSAGE + 8; public static final int MESSAGE_VISITORINFO = DOCUMENT_MESSAGE + 9; - public static final int MESSAGE_SEARCHRESULT = DOCUMENT_MESSAGE + 11; - public static final int MESSAGE_DOCUMENTSUMMARY = DOCUMENT_MESSAGE + 14; + // SearchResult and DocumentSummary messages were replaced by QueryResult message in 2010. + // public static final int MESSAGE_SEARCHRESULT = DOCUMENT_MESSAGE + 11; + // public static final int MESSAGE_DOCUMENTSUMMARY = DOCUMENT_MESSAGE + 14; public static final int MESSAGE_MAPVISITOR = DOCUMENT_MESSAGE + 15; public static final int MESSAGE_GETBUCKETSTATE = DOCUMENT_MESSAGE + 18; public static final int MESSAGE_STATBUCKET = DOCUMENT_MESSAGE + 19; @@ -70,8 +71,9 @@ public class DocumentProtocol implements Protocol { public static final int REPLY_CREATEVISITOR = DOCUMENT_REPLY + 7; public static final int REPLY_DESTROYVISITOR = DOCUMENT_REPLY + 8; public static final int REPLY_VISITORINFO = DOCUMENT_REPLY + 9; - public static final int REPLY_SEARCHRESULT = DOCUMENT_REPLY + 11; - public static final int REPLY_DOCUMENTSUMMARY = DOCUMENT_REPLY + 14; + // SearchResult and DocumentSummary replies were replaced by QueryResult reply in 2010. + // public static final int REPLY_SEARCHRESULT = DOCUMENT_REPLY + 11; + // public static final int REPLY_DOCUMENTSUMMARY = DOCUMENT_REPLY + 14; public static final int REPLY_MAPVISITOR = DOCUMENT_REPLY + 15; public static final int REPLY_GETBUCKETSTATE = DOCUMENT_REPLY + 18; public static final int REPLY_STATBUCKET = DOCUMENT_REPLY + 19; @@ -282,7 +284,6 @@ public class DocumentProtocol implements Protocol { putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories60.CreateVisitorMessageFactory(), from6); putRoutableFactory(MESSAGE_DESTROYVISITOR, new RoutableFactories60.DestroyVisitorMessageFactory(), from6); putRoutableFactory(MESSAGE_DOCUMENTLIST, new RoutableFactories60.DocumentListMessageFactory(), from6); - putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, new RoutableFactories60.DocumentSummaryMessageFactory(), from6); putRoutableFactory(MESSAGE_EMPTYBUCKETS, new RoutableFactories60.EmptyBucketsMessageFactory(), from6); putRoutableFactory(MESSAGE_GETBUCKETLIST, new RoutableFactories60.GetBucketListMessageFactory(), from6); putRoutableFactory(MESSAGE_GETBUCKETSTATE, new RoutableFactories60.GetBucketStateMessageFactory(), from6); @@ -292,7 +293,6 @@ public class DocumentProtocol implements Protocol { putRoutableFactory(MESSAGE_QUERYRESULT, new RoutableFactories60.QueryResultMessageFactory(), from6); putRoutableFactory(MESSAGE_REMOVEDOCUMENT, new RoutableFactories60.RemoveDocumentMessageFactory(), from6); putRoutableFactory(MESSAGE_REMOVELOCATION, new RoutableFactories60.RemoveLocationMessageFactory(), from6); - putRoutableFactory(MESSAGE_SEARCHRESULT, new RoutableFactories60.SearchResultMessageFactory(), from6); putRoutableFactory(MESSAGE_STATBUCKET, new RoutableFactories60.StatBucketMessageFactory(), from6); putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories60.UpdateDocumentMessageFactory(), from6); putRoutableFactory(MESSAGE_VISITORINFO, new RoutableFactories60.VisitorInfoMessageFactory(), from6); @@ -300,7 +300,6 @@ public class DocumentProtocol implements Protocol { putRoutableFactory(REPLY_DESTROYVISITOR, new RoutableFactories60.DestroyVisitorReplyFactory(), from6); putRoutableFactory(REPLY_DOCUMENTIGNORED, new RoutableFactories60.DocumentIgnoredReplyFactory(), from6); putRoutableFactory(REPLY_DOCUMENTLIST, new RoutableFactories60.DocumentListReplyFactory(), from6); - putRoutableFactory(REPLY_DOCUMENTSUMMARY, new RoutableFactories60.DocumentSummaryReplyFactory(), from6); putRoutableFactory(REPLY_EMPTYBUCKETS, new RoutableFactories60.EmptyBucketsReplyFactory(), from6); putRoutableFactory(REPLY_GETBUCKETLIST, new RoutableFactories60.GetBucketListReplyFactory(), from6); putRoutableFactory(REPLY_GETBUCKETSTATE, new RoutableFactories60.GetBucketStateReplyFactory(), from6); @@ -310,7 +309,6 @@ public class DocumentProtocol implements Protocol { putRoutableFactory(REPLY_QUERYRESULT, new RoutableFactories60.QueryResultReplyFactory(), from6); putRoutableFactory(REPLY_REMOVEDOCUMENT, new RoutableFactories60.RemoveDocumentReplyFactory(), from6); putRoutableFactory(REPLY_REMOVELOCATION, new RoutableFactories60.RemoveLocationReplyFactory(), from6); - putRoutableFactory(REPLY_SEARCHRESULT, new RoutableFactories60.SearchResultReplyFactory(), from6); putRoutableFactory(REPLY_STATBUCKET, new RoutableFactories60.StatBucketReplyFactory(), from6); putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories60.UpdateDocumentReplyFactory(), from6); putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories60.UpdateDocumentReplyFactory(), from6); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentSummaryMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentSummaryMessage.java deleted file mode 100644 index 4866579a977..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentSummaryMessage.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol; - -import com.yahoo.vdslib.DocumentSummary; - -public class DocumentSummaryMessage extends VisitorMessage { - - private DocumentSummary documentSummary = null; - - public void setDocumentSummary(DocumentSummary summary) { - documentSummary = summary; - } - - public DocumentSummary getResult() { - return documentSummary; - } - - @Override - public DocumentReply createReply() { - return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY); - } - - @Override - public int getType() { - return DocumentProtocol.MESSAGE_DOCUMENTSUMMARY; - } -} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java index 9812f214066..3824da32d4e 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java @@ -344,34 +344,6 @@ public abstract class RoutableFactories60 { } } - public static class DocumentSummaryMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - DocumentSummaryMessage msg = new DocumentSummaryMessage(); - msg.setDocumentSummary(new DocumentSummary(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - return false; // not supported - } - } - - public static class DocumentSummaryReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - public static class EmptyBucketsMessageFactory extends DocumentMessageFactory { @Override @@ -728,21 +700,6 @@ public abstract class RoutableFactories60 { } } - public static class SearchResultMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - SearchResultMessage msg = new SearchResultMessage(); - msg.setSearchResult(new SearchResult(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - return false; // not supported - } - } - public static class QueryResultMessageFactory extends DocumentMessageFactory { @Override @@ -759,19 +716,6 @@ public abstract class RoutableFactories60 { } } - public static class SearchResultReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - public static class QueryResultReplyFactory extends DocumentReplyFactory { @Override diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchResultMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchResultMessage.java deleted file mode 100644 index 570aafd49e6..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchResultMessage.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol; - -import com.yahoo.vdslib.SearchResult; - -public class SearchResultMessage extends VisitorMessage { - - private SearchResult searchResult = null; - - public SearchResult getResult() { - return searchResult; - } - - public void setSearchResult(SearchResult result) { - searchResult = result; - } - - @Override - public DocumentReply createReply() { - return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT); - } - - @Override - public int getType() { - return DocumentProtocol.MESSAGE_SEARCHRESULT; - } -} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java index 97cef53695a..22650fcdbf8 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java @@ -20,7 +20,6 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.DocumentReply; import com.yahoo.documentapi.messagebus.protocol.DocumentState; -import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage; import com.yahoo.documentapi.messagebus.protocol.GetBucketListMessage; import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; @@ -34,7 +33,6 @@ import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply; import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage; -import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; import com.yahoo.documentapi.messagebus.protocol.StatBucketMessage; import com.yahoo.documentapi.messagebus.protocol.StatBucketReply; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; @@ -69,7 +67,6 @@ public class Messages60TestCase extends MessagesTestBase { out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage()); out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage()); out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage()); - out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage()); out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage()); out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage()); out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage()); @@ -79,7 +76,6 @@ public class Messages60TestCase extends MessagesTestBase { out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage()); out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage()); out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage()); - out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage()); out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage()); out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage()); out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage()); @@ -87,7 +83,6 @@ public class Messages60TestCase extends MessagesTestBase { out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply()); out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply()); out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply()); - out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply()); out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply()); out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply()); out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply()); @@ -97,7 +92,6 @@ public class Messages60TestCase extends MessagesTestBase { out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply()); out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply()); out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply()); - out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply()); out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply()); out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply()); out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply()); @@ -324,14 +318,6 @@ public class Messages60TestCase extends MessagesTestBase { } } - public class testDocumentSummaryReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY); - } - } - public class testEmptyBucketsReply implements RunnableTest { @Override @@ -392,65 +378,6 @@ public class Messages60TestCase extends MessagesTestBase { } } - public class testDocumentSummaryMessage implements RunnableTest { - - @Override - public void run() { - Routable routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); - assertTrue(routable instanceof DocumentSummaryMessage); - - DocumentSummaryMessage msg = (DocumentSummaryMessage) routable; - assertEquals(0, msg.getResult().getSummaryCount()); - - routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); - assertTrue(routable instanceof DocumentSummaryMessage); - - msg = (DocumentSummaryMessage) routable; - assertEquals(2, msg.getResult().getSummaryCount()); - com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0); - assertEquals("doc1", s.getDocId()); - byte[] b = s.getSummary(); - assertEquals(8, b.length); - byte[] c = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'}; - for (int i = 0; i < b.length; i++) { - assertEquals(c[i], b[i]); - } - - s = msg.getResult().getSummary(1); - assertEquals("aoc17", s.getDocId()); - b = s.getSummary(); - assertEquals(9, b.length); - byte[] d = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'}; - for (int i = 0; i < b.length; i++) { - assertEquals(d[i], b[i]); - } - routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); - assertTrue(routable instanceof DocumentSummaryMessage); - - msg = (DocumentSummaryMessage) routable; - assertEquals(2, msg.getResult().getSummaryCount()); - - s = msg.getResult().getSummary(0); - assertEquals("aoc17", s.getDocId()); - b = s.getSummary(); - assertEquals(9, b.length); - byte[] e = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'}; - for (int i = 0; i < b.length; i++) { - assertEquals(e[i], b[i]); - } - - s = msg.getResult().getSummary(1); - assertEquals("doc1", s.getDocId()); - b = s.getSummary(); - assertEquals(8, b.length); - byte[] f = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'}; - for (int i = 0; i < b.length; i++) { - assertEquals(f[i], b[i]); - } - } - } - - public class testGetDocumentMessage implements RunnableTest { @Override @@ -521,78 +448,6 @@ public class Messages60TestCase extends MessagesTestBase { } } - public class testSearchResultMessage implements RunnableTest { - - @Override - public void run() throws Exception { - Routable routable = deserialize("SearchResultMessage-1", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - SearchResultMessage msg = (SearchResultMessage)routable; - assertEquals(0, msg.getResult().getHitCount()); - - routable = deserialize("SearchResultMessage-2", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - - routable = deserialize("SearchResultMessage-3", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - - routable = deserialize("SearchResultMessage-4", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(3, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; - for (int i = 0; i < b.length; i++) { - assertEquals(e[i], b[i]); - } - h = msg.getResult().getHit(1); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; - for (int i = 0; i < b.length; i++) { - assertEquals(d[i], b[i]); - } - h = msg.getResult().getHit(2); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(90.0, h.getRank(), 1E-6); - assertEquals("doc18", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; - for (int i = 0; i < b.length; i++) { - assertEquals(c[i], b[i]); - } - } - } - private static String CONDITION_STRING = "There's just one condition"; public class testPutDocumentMessage implements RunnableTest { @@ -750,14 +605,6 @@ public class Messages60TestCase extends MessagesTestBase { } } - public class testSearchResultReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT); - } - } - public class testStatBucketReply implements RunnableTest { @Override diff --git a/documentapi/src/tests/messages/messages60test.cpp b/documentapi/src/tests/messages/messages60test.cpp index 93c7b5ef7cb..12cecefb072 100644 --- a/documentapi/src/tests/messages/messages60test.cpp +++ b/documentapi/src/tests/messages/messages60test.cpp @@ -39,7 +39,6 @@ Messages60Test::Messages60Test() putTest(DocumentProtocol::MESSAGE_CREATEVISITOR, TEST_METHOD(Messages60Test::testCreateVisitorMessage)); putTest(DocumentProtocol::MESSAGE_DESTROYVISITOR, TEST_METHOD(Messages60Test::testDestroyVisitorMessage)); putTest(DocumentProtocol::MESSAGE_DOCUMENTLIST, TEST_METHOD(Messages60Test::testDocumentListMessage)); - putTest(DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, TEST_METHOD(Messages60Test::testDocumentSummaryMessage)); putTest(DocumentProtocol::MESSAGE_EMPTYBUCKETS, TEST_METHOD(Messages60Test::testEmptyBucketsMessage)); putTest(DocumentProtocol::MESSAGE_GETBUCKETLIST, TEST_METHOD(Messages60Test::testGetBucketListMessage)); putTest(DocumentProtocol::MESSAGE_GETBUCKETSTATE, TEST_METHOD(Messages60Test::testGetBucketStateMessage)); @@ -49,7 +48,6 @@ Messages60Test::Messages60Test() putTest(DocumentProtocol::MESSAGE_QUERYRESULT, TEST_METHOD(Messages60Test::testQueryResultMessage)); putTest(DocumentProtocol::MESSAGE_REMOVEDOCUMENT, TEST_METHOD(Messages60Test::testRemoveDocumentMessage)); putTest(DocumentProtocol::MESSAGE_REMOVELOCATION, TEST_METHOD(Messages60Test::testRemoveLocationMessage)); - putTest(DocumentProtocol::MESSAGE_SEARCHRESULT, TEST_METHOD(Messages60Test::testSearchResultMessage)); putTest(DocumentProtocol::MESSAGE_STATBUCKET, TEST_METHOD(Messages60Test::testStatBucketMessage)); putTest(DocumentProtocol::MESSAGE_UPDATEDOCUMENT, TEST_METHOD(Messages60Test::testUpdateDocumentMessage)); putTest(DocumentProtocol::MESSAGE_VISITORINFO, TEST_METHOD(Messages60Test::testVisitorInfoMessage)); @@ -58,7 +56,6 @@ Messages60Test::Messages60Test() putTest(DocumentProtocol::REPLY_DESTROYVISITOR, TEST_METHOD(Messages60Test::testDestroyVisitorReply)); putTest(DocumentProtocol::REPLY_DOCUMENTIGNORED, TEST_METHOD(Messages60Test::testDocumentIgnoredReply)); putTest(DocumentProtocol::REPLY_DOCUMENTLIST, TEST_METHOD(Messages60Test::testDocumentListReply)); - putTest(DocumentProtocol::REPLY_DOCUMENTSUMMARY, TEST_METHOD(Messages60Test::testDocumentSummaryReply)); putTest(DocumentProtocol::REPLY_EMPTYBUCKETS, TEST_METHOD(Messages60Test::testEmptyBucketsReply)); putTest(DocumentProtocol::REPLY_GETBUCKETLIST, TEST_METHOD(Messages60Test::testGetBucketListReply)); putTest(DocumentProtocol::REPLY_GETBUCKETSTATE, TEST_METHOD(Messages60Test::testGetBucketStateReply)); @@ -68,7 +65,6 @@ Messages60Test::Messages60Test() putTest(DocumentProtocol::REPLY_QUERYRESULT, TEST_METHOD(Messages60Test::testQueryResultReply)); putTest(DocumentProtocol::REPLY_REMOVEDOCUMENT, TEST_METHOD(Messages60Test::testRemoveDocumentReply)); putTest(DocumentProtocol::REPLY_REMOVELOCATION, TEST_METHOD(Messages60Test::testRemoveLocationReply)); - putTest(DocumentProtocol::REPLY_SEARCHRESULT, TEST_METHOD(Messages60Test::testSearchResultReply)); putTest(DocumentProtocol::REPLY_STATBUCKET, TEST_METHOD(Messages60Test::testStatBucketReply)); putTest(DocumentProtocol::REPLY_UPDATEDOCUMENT, TEST_METHOD(Messages60Test::testUpdateDocumentReply)); putTest(DocumentProtocol::REPLY_VISITORINFO, TEST_METHOD(Messages60Test::testVisitorInfoReply)); @@ -269,65 +265,6 @@ Messages60Test::testRemoveLocationMessage() bool -Messages60Test::testDocumentSummaryMessage() -{ - DocumentSummaryMessage srm; - EXPECT_EQUAL(srm.hasSequenceId(), false); - EXPECT_EQUAL(srm.getSummaryCount(), size_t(0)); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(12), serialize("DocumentSummaryMessage-1", srm)); - - mbus::Routable::UP routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - DocumentSummaryMessage * dm = static_cast<DocumentSummaryMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSummaryCount(), size_t(0)); - - srm.addSummary("doc1", "summary1", 8); - srm.addSummary("aoc17", "summary45", 9); - - const void *summary(NULL); - const char *docId(NULL); - size_t sz(0); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-2", srm)); - routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<DocumentSummaryMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); - dm->getSummary(0, docId, summary, sz); - EXPECT_EQUAL(sz, 8u); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - EXPECT_EQUAL(memcmp("summary1", summary, sz), 0); - dm->getSummary(1, docId, summary, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(strcmp("aoc17", docId), 0); - EXPECT_EQUAL(memcmp("summary45", summary, sz), 0); - - srm.sort(); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-3", srm)); - routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<DocumentSummaryMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); - dm->getSummary(0, docId, summary, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(strcmp("aoc17", docId), 0); - EXPECT_EQUAL(memcmp("summary45", summary, sz), 0); - dm->getSummary(1, docId, summary, sz); - EXPECT_EQUAL(sz, 8u); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - EXPECT_EQUAL(memcmp("summary1", summary, sz), 0); - return true; -} - -bool Messages60Test::testGetDocumentMessage() { GetDocumentMessage tmp(document::DocumentId("id:ns:testdoc::"), "foo bar"); @@ -561,133 +498,6 @@ Messages60Test::testRemoveDocumentReply() } bool -Messages60Test::testSearchResultMessage() -{ - SearchResultMessage srm; - EXPECT_EQUAL(srm.getSequenceId(), 0u); - EXPECT_EQUAL(srm.getHitCount(), 0u); - EXPECT_EQUAL(srm.getAggregatorList().getSerializedSize(), 4u); - EXPECT_EQUAL(srm.vdslib::SearchResult::getSerializedSize(), 20u); - EXPECT_EQUAL(srm.getSerializedSize(), 20u); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(24), serialize("SearchResultMessage-1", srm)); - - mbus::Routable::UP routable = deserialize("SearchResultMessage-1", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - SearchResultMessage * dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSequenceId(), size_t(0)); - EXPECT_EQUAL(dm->getHitCount(), size_t(0)); - - srm.addHit(0, "doc1", 89); - srm.addHit(1, "doc17", 109); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-2", srm)); - routable = deserialize("SearchResultMessage-2", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(2)); - const char *docId; - SearchResultMessage::RankType rank; - dm->getHit(0, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dm->getHit(1, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - - srm.sort(); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-3", srm)); - routable = deserialize("SearchResultMessage-3", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(2)); - dm->getHit(0, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dm->getHit(1, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - - SearchResultMessage srm2; - srm2.addHit(0, "doc1", 89, "sortdata2", 9); - srm2.addHit(1, "doc17", 109, "sortdata1", 9); - srm2.addHit(2, "doc18", 90, "sortdata3", 9); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-4", srm2)); - routable = deserialize("SearchResultMessage-4", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(3)); - dm->getHit(0, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dm->getHit(1, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dm->getHit(2, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - - srm2.sort(); - const void *buf; - size_t sz; - srm2.getHit(0, docId, rank); - srm2.getSortBlob(0, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - srm2.getHit(1, docId, rank); - srm2.getSortBlob(1, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - srm2.getHit(2, docId, rank); - srm2.getSortBlob(2, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-5", srm2)); - routable = deserialize("SearchResultMessage-5", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(3)); - dm->getHit(0, docId, rank); - dm->getSortBlob(0, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dm->getHit(1, docId, rank); - dm->getSortBlob(1, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dm->getHit(2, docId, rank); - dm->getSortBlob(2, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - return true; -} - -bool Messages60Test::testUpdateDocumentMessage() { const DocumentTypeRepo & repo = getTypeRepo(); @@ -913,12 +723,6 @@ Messages60Test::testDocumentListReply() } bool -Messages60Test::testDocumentSummaryReply() -{ - return tryVisitorReply("DocumentSummaryReply", DocumentProtocol::REPLY_DOCUMENTSUMMARY); -} - -bool Messages60Test::testGetDocumentReply() { document::Document::SP doc = @@ -947,12 +751,6 @@ Messages60Test::testMapVisitorReply() } bool -Messages60Test::testSearchResultReply() -{ - return tryVisitorReply("SearchResultReply", DocumentProtocol::REPLY_SEARCHRESULT); -} - -bool Messages60Test::testStatBucketReply() { StatBucketReply msg; diff --git a/documentapi/src/tests/messages/messages60test.h b/documentapi/src/tests/messages/messages60test.h index 1eb3e8e248f..4a2a3f98fad 100644 --- a/documentapi/src/tests/messages/messages60test.h +++ b/documentapi/src/tests/messages/messages60test.h @@ -24,7 +24,6 @@ public: bool testDocumentListMessage(); bool testDocumentListReply(); bool testDocumentSummaryMessage(); - bool testDocumentSummaryReply(); bool testEmptyBucketsMessage(); bool testEmptyBucketsReply(); bool testGetBucketListMessage(); @@ -44,7 +43,6 @@ public: bool testRemoveLocationMessage(); bool testRemoveLocationReply(); bool testSearchResultMessage(); - bool testSearchResultReply(); bool testStatBucketMessage(); bool testStatBucketReply(); bool testUpdateDocumentMessage(); diff --git a/documentapi/src/vespa/documentapi/documentapi.h b/documentapi/src/vespa/documentapi/documentapi.h index 311199c94cd..b784a63a642 100644 --- a/documentapi/src/vespa/documentapi/documentapi.h +++ b/documentapi/src/vespa/documentapi/documentapi.h @@ -11,9 +11,7 @@ #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> #include <vespa/documentapi/messagebus/messages/feedreply.h> #include <vespa/documentapi/messagebus/messages/updatedocumentmessage.h> -#include <vespa/documentapi/messagebus/messages/searchresultmessage.h> #include <vespa/documentapi/messagebus/messages/visitor.h> -#include <vespa/documentapi/messagebus/messages/documentsummarymessage.h> #include <vespa/documentapi/messagebus/messages/wrongdistributionreply.h> #include <vespa/documentapi/messagebus/messages/getbucketlistmessage.h> #include <vespa/documentapi/messagebus/messages/getbucketlistreply.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index d751d11177c..1eb50bba714 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -49,7 +49,6 @@ DocumentProtocol::DocumentProtocol(std::shared_ptr<const DocumentTypeRepo> repo, putRoutableFactory(MESSAGE_CREATEVISITOR, std::make_shared<RoutableFactories60::CreateVisitorMessageFactory>(), from6); putRoutableFactory(MESSAGE_DESTROYVISITOR, std::make_shared<RoutableFactories60::DestroyVisitorMessageFactory>(), from6); putRoutableFactory(MESSAGE_DOCUMENTLIST, std::make_shared<RoutableFactories60::DocumentListMessageFactory>(*_repo), from6); - putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, std::make_shared<RoutableFactories60::DocumentSummaryMessageFactory>(), from6); putRoutableFactory(MESSAGE_EMPTYBUCKETS, std::make_shared<RoutableFactories60::EmptyBucketsMessageFactory>(), from6); putRoutableFactory(MESSAGE_GETBUCKETLIST, std::make_shared<RoutableFactories60::GetBucketListMessageFactory>(), from6); putRoutableFactory(MESSAGE_GETBUCKETSTATE, std::make_shared<RoutableFactories60::GetBucketStateMessageFactory>(), from6); @@ -59,7 +58,6 @@ DocumentProtocol::DocumentProtocol(std::shared_ptr<const DocumentTypeRepo> repo, putRoutableFactory(MESSAGE_QUERYRESULT, std::make_shared<RoutableFactories60::QueryResultMessageFactory>(), from6); putRoutableFactory(MESSAGE_REMOVEDOCUMENT, std::make_shared<RoutableFactories60::RemoveDocumentMessageFactory>(), from6); putRoutableFactory(MESSAGE_REMOVELOCATION, std::make_shared<RoutableFactories60::RemoveLocationMessageFactory>(*_repo), from6); - putRoutableFactory(MESSAGE_SEARCHRESULT, std::make_shared<RoutableFactories60::SearchResultMessageFactory>(), from6); putRoutableFactory(MESSAGE_STATBUCKET, std::make_shared<RoutableFactories60::StatBucketMessageFactory>(), from6); putRoutableFactory(MESSAGE_UPDATEDOCUMENT, std::make_shared<RoutableFactories60::UpdateDocumentMessageFactory>(*_repo), from6); putRoutableFactory(MESSAGE_VISITORINFO, std::make_shared<RoutableFactories60::VisitorInfoMessageFactory>(), from6); @@ -67,7 +65,6 @@ DocumentProtocol::DocumentProtocol(std::shared_ptr<const DocumentTypeRepo> repo, putRoutableFactory(REPLY_DESTROYVISITOR, std::make_shared<RoutableFactories60::DestroyVisitorReplyFactory>(), from6); putRoutableFactory(REPLY_DOCUMENTIGNORED, std::make_shared<RoutableFactories60::DocumentIgnoredReplyFactory>(), from6); putRoutableFactory(REPLY_DOCUMENTLIST, std::make_shared<RoutableFactories60::DocumentListReplyFactory>(), from6); - putRoutableFactory(REPLY_DOCUMENTSUMMARY, std::make_shared<RoutableFactories60::DocumentSummaryReplyFactory>(), from6); putRoutableFactory(REPLY_EMPTYBUCKETS, std::make_shared<RoutableFactories60::EmptyBucketsReplyFactory>(), from6); putRoutableFactory(REPLY_GETBUCKETLIST, std::make_shared<RoutableFactories60::GetBucketListReplyFactory>(), from6); putRoutableFactory(REPLY_GETBUCKETSTATE, std::make_shared<RoutableFactories60::GetBucketStateReplyFactory>(), from6); @@ -77,7 +74,6 @@ DocumentProtocol::DocumentProtocol(std::shared_ptr<const DocumentTypeRepo> repo, putRoutableFactory(REPLY_QUERYRESULT, std::make_shared<RoutableFactories60::QueryResultReplyFactory>(), from6); putRoutableFactory(REPLY_REMOVEDOCUMENT, std::make_shared<RoutableFactories60::RemoveDocumentReplyFactory>(), from6); putRoutableFactory(REPLY_REMOVELOCATION, std::make_shared<RoutableFactories60::RemoveLocationReplyFactory>(), from6); - putRoutableFactory(REPLY_SEARCHRESULT, std::make_shared<RoutableFactories60::SearchResultReplyFactory>(), from6); putRoutableFactory(REPLY_STATBUCKET, std::make_shared<RoutableFactories60::StatBucketReplyFactory>(), from6); putRoutableFactory(REPLY_UPDATEDOCUMENT, std::make_shared<RoutableFactories60::UpdateDocumentReplyFactory>(), from6); putRoutableFactory(REPLY_VISITORINFO, std::make_shared<RoutableFactories60::VisitorInfoReplyFactory>(), from6); diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h index 9f0d7253335..d91d355c567 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h @@ -55,9 +55,10 @@ public: MESSAGE_CREATEVISITOR = DOCUMENT_MESSAGE + 7, MESSAGE_DESTROYVISITOR = DOCUMENT_MESSAGE + 8, MESSAGE_VISITORINFO = DOCUMENT_MESSAGE + 9, - MESSAGE_SEARCHRESULT = DOCUMENT_MESSAGE + 11, + // SearchResult and DocumentSummary messages were replaced by QueryResult message in 2010. + // MESSAGE_SEARCHRESULT = DOCUMENT_MESSAGE + 11, //MESSAGE_MULTIOPERATION = DOCUMENT_MESSAGE + 13, - MESSAGE_DOCUMENTSUMMARY = DOCUMENT_MESSAGE + 14, + // MESSAGE_DOCUMENTSUMMARY = DOCUMENT_MESSAGE + 14, MESSAGE_MAPVISITOR = DOCUMENT_MESSAGE + 15, MESSAGE_GETBUCKETSTATE = DOCUMENT_MESSAGE + 18, MESSAGE_STATBUCKET = DOCUMENT_MESSAGE + 19, @@ -78,9 +79,10 @@ public: REPLY_CREATEVISITOR = DOCUMENT_REPLY + 7, REPLY_DESTROYVISITOR = DOCUMENT_REPLY + 8, REPLY_VISITORINFO = DOCUMENT_REPLY + 9, - REPLY_SEARCHRESULT = DOCUMENT_REPLY + 11, + // SearchResult and DocumentSummary replies were replaced by QueryResult reply in 2010. + // REPLY_SEARCHRESULT = DOCUMENT_REPLY + 11, //REPLY_MULTIOPERATION = DOCUMENT_REPLY + 13, - REPLY_DOCUMENTSUMMARY = DOCUMENT_REPLY + 14, + // REPLY_DOCUMENTSUMMARY = DOCUMENT_REPLY + 14, REPLY_MAPVISITOR = DOCUMENT_REPLY + 15, REPLY_GETBUCKETSTATE = DOCUMENT_REPLY + 18, REPLY_STATBUCKET = DOCUMENT_REPLY + 19, diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt index ddbe66fc22a..d906166b6df 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_library(documentapi_documentapimessages OBJECT documentmessage.cpp documentreply.cpp documentstate.cpp - documentsummarymessage.cpp emptybucketsmessage.cpp feedanswer.cpp feedmessage.cpp @@ -21,7 +20,6 @@ vespa_add_library(documentapi_documentapimessages OBJECT removedocumentmessage.cpp removedocumentreply.cpp removelocationmessage.cpp - searchresultmessage.cpp statbucketmessage.cpp statbucketreply.cpp testandsetmessage.cpp diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp deleted file mode 100644 index 6be241b2e1d..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "documentsummarymessage.h" - -using vdslib::DocumentSummary; - -namespace documentapi { - -DocumentSummaryMessage::DocumentSummaryMessage(const DocumentSummary & sr) : - VisitorMessage(), - DocumentSummary(sr) -{ - // empty -} - -DocumentSummaryMessage::DocumentSummaryMessage() : - VisitorMessage(), - DocumentSummary() -{ - // empty -} - -DocumentReply::UP -DocumentSummaryMessage::doCreateReply() const -{ - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTSUMMARY)); -} - -uint32_t -DocumentSummaryMessage::getApproxSize() const -{ - return DocumentSummary::getSerializedSize(); -} - -uint32_t -DocumentSummaryMessage::getType() const -{ - return DocumentProtocol::MESSAGE_DOCUMENTSUMMARY; -} - -} - diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.h deleted file mode 100644 index 2c8149ab058..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include <vespa/vdslib/container/documentsummary.h> - -namespace documentapi { - -class DocumentSummaryMessage : public VisitorMessage, - public vdslib::DocumentSummary { -protected: - // Implements VisitorMessage. - DocumentReply::UP doCreateReply() const override; - -public: - /** - * Convenience typedef. - */ - using UP = std::unique_ptr<DocumentSummaryMessage>; - using SP = std::shared_ptr<DocumentSummaryMessage>; - - /** - * Constructs a new document message with no content. - */ - DocumentSummaryMessage(); - - /** - * Constructs a new document message with summary comment. - * - * @param summary The document summary to contain. - */ - DocumentSummaryMessage(const vdslib::DocumentSummary &summary); - uint32_t getApproxSize() const override; - uint32_t getType() const override; - string toString() const override { return "documentsummarymessage"; } -}; - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp deleted file mode 100644 index 8e25e70d749..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "searchresultmessage.h" - -using vdslib::SearchResult; - -namespace documentapi { - -SearchResultMessage::SearchResultMessage() : - VisitorMessage(), - SearchResult() -{ - // empty -} - -SearchResultMessage::SearchResultMessage(SearchResult &&result) : - VisitorMessage(), - SearchResult(std::move(result)) -{ - // empty -} - -DocumentReply::UP -SearchResultMessage::doCreateReply() const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_SEARCHRESULT); -} - -uint32_t -SearchResultMessage::getApproxSize() const -{ - return SearchResult::getSerializedSize(); -} - -uint32_t -SearchResultMessage::getType() const -{ - return DocumentProtocol::MESSAGE_SEARCHRESULT; -} - -} - diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h deleted file mode 100644 index d22f8197534..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include <vespa/vdslib/container/searchresult.h> - -namespace documentapi { - -class SearchResultMessage : public VisitorMessage, - public vdslib::SearchResult { -protected: - DocumentReply::UP doCreateReply() const override; - -public: - using UP = std::unique_ptr<SearchResultMessage>; - using SP = std::shared_ptr<SearchResultMessage>; - - SearchResultMessage(); - SearchResultMessage(vdslib::SearchResult &&result); - - uint32_t getApproxSize() const override; - uint32_t getType() const override; - string toString() const override { return "searchresultmessage"; } -}; - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp index 1dce7ff281f..508bb25c907 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp @@ -258,37 +258,6 @@ RoutableFactories60::DocumentListReplyFactory::doEncode(const DocumentReply &, v } DocumentMessage::UP -RoutableFactories60::DocumentSummaryMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<DocumentSummaryMessage>(); - - msg->deserialize(buf); - - return msg; -} - -bool -RoutableFactories60::DocumentSummaryMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const DocumentSummaryMessage &msg = static_cast<const DocumentSummaryMessage&>(obj); - msg.serialize(buf); - - return true; -} - -DocumentReply::UP -RoutableFactories60::DocumentSummaryReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTSUMMARY); -} - -bool -RoutableFactories60::DocumentSummaryReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -DocumentMessage::UP RoutableFactories60::EmptyBucketsMessageFactory::doDecode(document::ByteBuffer &buf) const { auto msg = std::make_unique<EmptyBucketsMessage>(); @@ -642,23 +611,6 @@ RoutableFactories60::RemoveLocationReplyFactory::doEncode(const DocumentReply &, } DocumentMessage::UP -RoutableFactories60::SearchResultMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<SearchResultMessage>(); - msg->deserialize(buf); - return msg; -} - -bool -RoutableFactories60::SearchResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const auto & msg = static_cast<const SearchResultMessage&>(obj); - msg.serialize(buf); - - return true; -} - -DocumentMessage::UP RoutableFactories60::QueryResultMessageFactory::doDecode(document::ByteBuffer &buf) const { auto msg = std::make_unique<QueryResultMessage>(); @@ -680,18 +632,6 @@ RoutableFactories60::QueryResultMessageFactory::doEncode(const DocumentMessage & } DocumentReply::UP -RoutableFactories60::SearchResultReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_SEARCHRESULT); -} - -bool -RoutableFactories60::SearchResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -DocumentReply::UP RoutableFactories60::QueryResultReplyFactory::doDecode(document::ByteBuffer &) const { return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_QUERYRESULT); diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h index b618d92a145..c0cbc4868eb 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h @@ -172,16 +172,6 @@ public: DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; }; - class DocumentSummaryMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class DocumentSummaryReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; class EmptyBucketsMessageFactory : public DocumentMessageFactory { protected: DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; @@ -282,16 +272,6 @@ public: DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; }; - class SearchResultMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class SearchResultReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; class StatBucketMessageFactory : public DocumentMessageFactory { virtual bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const; virtual string decodeBucketSpace(document::ByteBuffer&) const; diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-1.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-1.dat Binary files differdeleted file mode 100644 index 0107dd5f350..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-1.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-2.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-2.dat Binary files differdeleted file mode 100644 index 57187093f28..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-2.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-3.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-3.dat Binary files differdeleted file mode 100644 index 6a516d38d17..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-3.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryReply.dat Binary files differdeleted file mode 100644 index 16b1e4bc4ef..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-1.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-1.dat Binary files differdeleted file mode 100644 index 988f9fdab1f..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-1.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-2.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-2.dat Binary files differdeleted file mode 100644 index ac277d09643..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-2.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-3.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-3.dat Binary files differdeleted file mode 100644 index 03b49c8a0ac..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-3.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-4.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-4.dat Binary files differdeleted file mode 100644 index d52e574ea44..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-4.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-5.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-5.dat Binary files differdeleted file mode 100644 index e68654e9941..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-5.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultReply.dat Binary files differdeleted file mode 100644 index cce9c6f8d14..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-java-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/6.221-java-DocumentSummaryReply.dat Binary files differdeleted file mode 100644 index 16b1e4bc4ef..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-java-DocumentSummaryReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-java-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/6.221-java-SearchResultReply.dat Binary files differdeleted file mode 100644 index cce9c6f8d14..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-java-SearchResultReply.dat +++ /dev/null diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index e5aa47fe5c9..b37fe02226b 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -130,7 +130,7 @@ public class FileReceiver { moveFileToDestination(inprogressFile, file); } else { decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); - log.log(Level.FINE, () -> "compression type to use=" + compressionType); + log.log(Level.FINEST, () -> "compression type to use=" + compressionType); new FileReferenceCompressor(fileType, compressionType).decompress(inprogressFile, decompressedDir); moveFileToDestination(decompressedDir, fileReferenceDir); } @@ -230,7 +230,7 @@ public class FileReceiver { } private void receiveFileMeta(Request req) { - log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINEST, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); String fileName = req.parameters().get(1).asString(); Type type = FileReferenceData.Type.valueOf(req.parameters().get(2).asString()); @@ -281,7 +281,7 @@ public class FileReceiver { } private void receiveFileEof(Request req) { - log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINEST, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); int sessionId = req.parameters().get(1).asInt32(); long xxhash = req.parameters().get(2).asInt64(); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java index 8d6f9ea1af3..5ab1841486e 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java @@ -61,7 +61,7 @@ public class FileReferenceCompressor { } public void decompress(File inputFile, File outputDir) throws IOException { - log.log(Level.FINE, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); + log.log(Level.FINEST, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); try (ArchiveInputStream ais = new TarArchiveInputStream(decompressedInputStream(inputFile))) { decompress(ais, outputDir); } catch (IllegalArgumentException e) { @@ -121,7 +121,7 @@ public class FileReferenceCompressor { private OutputStream compressedOutputStream(File outputFile) throws IOException { switch (type) { case compressed: - log.log(Level.FINE, () -> "Compressing with compression type " + compressionType); + log.log(Level.FINEST, () -> "Compressing with compression type " + compressionType); return switch (compressionType) { case gzip -> new GZIPOutputStream(new FileOutputStream(outputFile)); case lz4 -> new LZ4BlockOutputStream(new FileOutputStream(outputFile)); @@ -137,7 +137,7 @@ public class FileReferenceCompressor { private InputStream decompressedInputStream(File inputFile) throws IOException { switch (type) { case compressed: - log.log(Level.FINE, () -> "Decompressing with compression type " + compressionType); + log.log(Level.FINEST, () -> "Decompressing with compression type " + compressionType); return switch (compressionType) { case gzip -> new GZIPInputStream(new FileInputStream(inputFile)); case lz4 -> new LZ4BlockInputStream(new FileInputStream(inputFile)); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 345a6f700fd..500adab72cf 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -380,7 +380,7 @@ public class Flags { public static final UnboundLongFlag ZOOKEEPER_BARRIER_WAIT_FOR_ALL_TIMEOUT = defineLongFlag( "zookeeper-barrier-wait-for-all-timeout", 1, - List.of("hmusum"), "2023-03-28", "2023-04-28", + List.of("hmusum"), "2023-03-28", "2023-05-28", "Time to wait for all barrier members after getting response from quorum number of member", "Takes effect on next config server container start", ZONE_ID); @@ -402,6 +402,18 @@ public class Flags { "allow-more-than-one-content-group-down", false, List.of("hmusum"), "2023-04-14", "2023-06-14", "Whether to enable possible configuration of letting more than one content group down", "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundBooleanFlag FAIL_DEPLOYMENT_ON_MISSING_CERTIFICATE_FILE = defineFeatureFlag( + "fail-on-missing-certificate-file", false, List.of("hmusum"), "2023-04-21", "2023-05-21", + "Whether to fail in controller when a submitted application package has no certificate files", + "Takes effect at redeployment", + ZONE_ID); + + public static final UnboundBooleanFlag NEW_IDDOC_LAYOUT = defineFeatureFlag( + "new_iddoc_layout", false, List.of("tokle", "bjorncs", "olaa"), "2023-04-24", "2023-05-31", + "Whether to use new identity document lauoyt", + "Takes effect on node reboot", HOSTNAME); /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroup.java deleted file mode 100644 index b98ad7a11bc..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroup.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.collections.Pair; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; - -import static com.yahoo.vespa.hosted.node.admin.container.ContainerStatsCollector.userHzToMicroSeconds; - -/** - * Read and write interface to the CGroup of a podman container. - * - * @author freva - */ -public interface CGroup { - - /** - * Returns quota and period values used for CPU scheduling. This serves as hard cap on CPU usage by allowing - * the CGroup to use up to {@code quota} each {@code period}. If uncapped, quota will be negative. - * - * @param containerId full container ID. - * @return CPU quota and period for the given container. Empty if CGroup for this container is not found. - */ - Optional<Pair<Integer, Integer>> cpuQuotaPeriod(ContainerId containerId); - - /** @return number of shares allocated to this CGroup for purposes of CPU time scheduling, empty if CGroup not found */ - OptionalInt cpuShares(ContainerId containerId); - - /** Update CPU quota and period for the given container ID, set quota to -1 value for unlimited */ - boolean updateCpuQuotaPeriod(NodeAgentContext context, ContainerId containerId, int cpuQuotaUs, int periodUs); - - boolean updateCpuShares(NodeAgentContext context, ContainerId containerId, int shares); - - Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException; - - /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */ - long memoryLimitInBytes(ContainerId containerId) throws IOException; - - /** @return The total amount of memory currently being used by the cgroup and its descendants. */ - long memoryUsageInBytes(ContainerId containerId) throws IOException; - - /** @return Number of bytes used to cache filesystem data, including tmpfs and shared memory. */ - long memoryCacheInBytes(ContainerId containerId) throws IOException; - - enum CpuStatField { - TOTAL_USAGE_USEC(null/* in a dedicated file */, "usage_usec"), - USER_USAGE_USEC("user", "user_usec"), - SYSTEM_USAGE_USEC("system", "system_usec"), - TOTAL_PERIODS("nr_periods", "nr_periods"), - THROTTLED_PERIODS("nr_throttled", "nr_throttled"), - THROTTLED_TIME_USEC("throttled_time", "throttled_usec"); - - private final String v1Name; - private final String v2Name; - CpuStatField(String v1Name, String v2Name) { - this.v1Name = v1Name; - this.v2Name = v2Name; - } - - long parseValueV1(String value) { - long longValue = Long.parseLong(value); - return switch (this) { - case THROTTLED_TIME_USEC, TOTAL_USAGE_USEC -> longValue / 1000; // Value in ns - case USER_USAGE_USEC, SYSTEM_USAGE_USEC -> userHzToMicroSeconds(longValue); - default -> longValue; - }; - } - - long parseValueV2(String value) { - return Long.parseLong(value); - } - - static Optional<CpuStatField> fromV1Field(String name) { - return Arrays.stream(values()) - .filter(field -> name.equals(field.v1Name)) - .findFirst(); - } - - static Optional<CpuStatField> fromV2Field(String name) { - return Arrays.stream(values()) - .filter(field -> name.equals(field.v2Name)) - .findFirst(); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1.java deleted file mode 100644 index 7607858ec85..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.collections.Pair; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.logging.Logger; -import java.util.stream.Stream; - -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.parseLong; - -/** - * Read and write interface to the CGroup V1 of a Podman container. - * - * @see <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/index.html">CGroups V1</a> - * @author freva - */ -public class CGroupV1 implements CGroup { - - private static final Logger logger = Logger.getLogger(CGroupV1.class.getName()); - - private final FileSystem fileSystem; - - public CGroupV1(FileSystem fileSystem) { - this.fileSystem = fileSystem; - } - - @Override - public Optional<Pair<Integer, Integer>> cpuQuotaPeriod(ContainerId containerId) { - OptionalInt quota = readCgroupsCpuInt(cfsQuotaPath(containerId)); - if (quota.isEmpty()) return Optional.empty(); - OptionalInt period = readCgroupsCpuInt(cfsPeriodPath(containerId)); - if (period.isEmpty()) return Optional.empty(); - return Optional.of(new Pair<>(quota.getAsInt(), period.getAsInt())); - } - - @Override - public OptionalInt cpuShares(ContainerId containerId) { - return readCgroupsCpuInt(sharesPath(containerId)); - } - - @Override - public boolean updateCpuQuotaPeriod(NodeAgentContext context, ContainerId containerId, int cpuQuotaUs, int periodUs) { - return writeCgroupsCpuInt(context, cfsQuotaPath(containerId), cpuQuotaUs) | - writeCgroupsCpuInt(context, cfsPeriodPath(containerId), periodUs); - } - - @Override - public boolean updateCpuShares(NodeAgentContext context, ContainerId containerId, int shares) { - return writeCgroupsCpuInt(context, sharesPath(containerId), shares); - } - - @Override - public Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException { - Map<CpuStatField, Long> stats = new HashMap<>(); - stats.put(CpuStatField.TOTAL_USAGE_USEC, parseLong(cpuacctPath(containerId).resolve("cpuacct.usage")) / 1000); - Stream.concat(Files.readAllLines(cpuacctPath(containerId).resolve("cpuacct.stat")).stream(), - Files.readAllLines(cpuacctPath(containerId).resolve("cpu.stat")).stream()) - .forEach(line -> { - String[] parts = line.split("\\s+"); - if (parts.length != 2) return; - CpuStatField.fromV1Field(parts[0]).ifPresent(field -> stats.put(field, field.parseValueV1(parts[1]))); - }); - return stats; - } - - @Override - public long memoryLimitInBytes(ContainerId containerId) throws IOException { - return parseLong(memoryPath(containerId).resolve("memory.limit_in_bytes")); - } - - @Override - public long memoryUsageInBytes(ContainerId containerId) throws IOException { - return parseLong(memoryPath(containerId).resolve("memory.usage_in_bytes")); - } - - @Override - public long memoryCacheInBytes(ContainerId containerId) throws IOException { - return parseLong(memoryPath(containerId).resolve("memory.stat"), "cache"); - } - - private Path cpuacctPath(ContainerId containerId) { - return fileSystem.getPath("/sys/fs/cgroup/cpuacct/machine.slice/libpod-" + containerId + ".scope"); - } - - private Path cpuPath(ContainerId containerId) { - return fileSystem.getPath("/sys/fs/cgroup/cpu/machine.slice/libpod-" + containerId + ".scope"); - } - - private Path memoryPath(ContainerId containerId) { - return fileSystem.getPath("/sys/fs/cgroup/memory/machine.slice/libpod-" + containerId + ".scope"); - } - - private UnixPath cfsQuotaPath(ContainerId containerId) { - return new UnixPath(cpuPath(containerId).resolve("cpu.cfs_quota_us")); - } - - private UnixPath cfsPeriodPath(ContainerId containerId) { - return new UnixPath(cpuPath(containerId).resolve("cpu.cfs_period_us")); - } - - private UnixPath sharesPath(ContainerId containerId) { - return new UnixPath(cpuPath(containerId).resolve("cpu.shares")); - } - - private static OptionalInt readCgroupsCpuInt(UnixPath unixPath) { - return unixPath.readUtf8FileIfExists() - .map(s -> OptionalInt.of(Integer.parseInt(s.strip()))) - .orElseGet(OptionalInt::empty); - } - - private static boolean writeCgroupsCpuInt(NodeAgentContext context, UnixPath unixPath, int value) { - int currentValue = readCgroupsCpuInt(unixPath).orElseThrow(); - if (currentValue == value) return false; - - context.recordSystemModification(logger, "Updating " + unixPath + " from " + currentValue + " to " + value); - unixPath.writeUtf8File(Integer.toString(value)); - return true; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java index 0c86829b96d..3cb34e066ff 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.container; import com.yahoo.collections.Pair; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; @@ -9,6 +10,7 @@ import java.io.IOException; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -17,23 +19,48 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Read and write interface to the CGroup V2 of a Podman container. + * Read and write interface to the cgroup v2 of a Podman container. * * @see <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html">CGroups V2</a> * @author freva */ -public class CGroupV2 implements CGroup { +public class CGroupV2 { private static final Logger logger = Logger.getLogger(CGroupV2.class.getName()); private static final String MAX = "max"; - private final FileSystem fileSystem; + private final Path rootCgroupPath; public CGroupV2(FileSystem fileSystem) { - this.fileSystem = fileSystem; - } - - @Override + this.rootCgroupPath = fileSystem.getPath("/sys/fs/cgroup"); + } + + /** + * Wraps {@code command} to ensure it is executed in the given cgroup. + * + * <p>WARNING: This method must be called only after vespa-cgexec has been installed.</p> + * + * @param cgroup The cgroup to execute the command in, e.g. /sys/fs/cgroup/system.slice/wireguard.scope. + * @param command The command to execute in the cgroup. + * @see #cgroupRootPath() + * @see #cgroupPath(ContainerId) + */ + public String[] wrapForExecutionIn(Path cgroup, String... command) { + String[] fullCommand = new String[3 + command.length]; + fullCommand[0] = Defaults.getDefaults().vespaHome() + "/bin/vespa-cgexec"; + fullCommand[1] = "-g"; + fullCommand[2] = cgroup.toString(); + System.arraycopy(command, 0, fullCommand, 3, command.length); + return fullCommand; + } + + /** + * Returns quota and period values used for CPU scheduling. This serves as hard cap on CPU usage by allowing + * the CGroupV2 to use up to {@code quota} each {@code period}. If uncapped, quota will be negative. + * + * @param containerId full container ID. + * @return CPU quota and period for the given container. Empty if CGroupV2 for this container is not found. + */ public Optional<Pair<Integer, Integer>> cpuQuotaPeriod(ContainerId containerId) { return cpuMaxPath(containerId).readUtf8FileIfExists() .map(s -> { @@ -42,60 +69,89 @@ public class CGroupV2 implements CGroup { }); } - @Override + /** @return number of shares allocated to this CGroupV2 for purposes of CPU time scheduling, empty if CGroupV2 not found */ public OptionalInt cpuShares(ContainerId containerId) { return cpuWeightPath(containerId).readUtf8FileIfExists() .map(s -> OptionalInt.of(weightToShares(Integer.parseInt(s.strip())))) .orElseGet(OptionalInt::empty); } - @Override + /** Update CPU quota and period for the given container ID, set quota to -1 value for unlimited */ public boolean updateCpuQuotaPeriod(NodeAgentContext context, ContainerId containerId, int cpuQuotaUs, int periodUs) { String wanted = String.format("%s %d", cpuQuotaUs < 0 ? MAX : cpuQuotaUs, periodUs); return writeCGroupsValue(context, cpuMaxPath(containerId), wanted); } - @Override public boolean updateCpuShares(NodeAgentContext context, ContainerId containerId, int shares) { return writeCGroupsValue(context, cpuWeightPath(containerId), Integer.toString(sharesToWeight(shares))); } - @Override + enum CpuStatField { + TOTAL_USAGE_USEC("usage_usec"), + USER_USAGE_USEC("user_usec"), + SYSTEM_USAGE_USEC("system_usec"), + TOTAL_PERIODS("nr_periods"), + THROTTLED_PERIODS("nr_throttled"), + THROTTLED_TIME_USEC("throttled_usec"); + + private final String name; + + CpuStatField(String name) { + this.name = name; + } + + long parseValue(String value) { + return Long.parseLong(value); + } + + static Optional<CpuStatField> fromField(String fieldName) { + return Arrays.stream(values()) + .filter(field -> fieldName.equals(field.name)) + .findFirst(); + } + } + public Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException { - return Files.readAllLines(cgroupRoot(containerId).resolve("cpu.stat")).stream() - .map(line -> line.split("\\s+")) - .filter(parts -> parts.length == 2) - .flatMap(parts -> CpuStatField.fromV2Field(parts[0]).stream().map(field -> new Pair<>(field, field.parseValueV2(parts[1])))) - .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); + return Files.readAllLines(cgroupPath(containerId).resolve("cpu.stat")).stream() + .map(line -> line.split("\\s+")) + .filter(parts -> parts.length == 2) + .flatMap(parts -> CpuStatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1])))) + .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); } - @Override + /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */ public long memoryLimitInBytes(ContainerId containerId) throws IOException { - String limit = Files.readString(cgroupRoot(containerId).resolve("memory.max")).strip(); + String limit = Files.readString(cgroupPath(containerId).resolve("memory.max")).strip(); return MAX.equals(limit) ? -1L : Long.parseLong(limit); } - @Override + /** @return The total amount of memory currently being used by the cgroup and its descendants. */ public long memoryUsageInBytes(ContainerId containerId) throws IOException { - return parseLong(cgroupRoot(containerId).resolve("memory.current")); + return parseLong(cgroupPath(containerId).resolve("memory.current")); } - @Override + /** @return Number of bytes used to cache filesystem data, including tmpfs and shared memory. */ public long memoryCacheInBytes(ContainerId containerId) throws IOException { - return parseLong(cgroupRoot(containerId).resolve("memory.stat"), "file"); + return parseLong(cgroupPath(containerId).resolve("memory.stat"), "file"); + } + + /** Returns the cgroup v2 mount point path (/sys/fs/cgroup). */ + public Path cgroupRootPath() { + return rootCgroupPath; } - private Path cgroupRoot(ContainerId containerId) { + /** Returns the cgroup directory of the Podman container, and which appears as the root cgroup within the container. */ + public Path cgroupPath(ContainerId containerId) { // crun path, runc path is without the 'container' directory - return fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-" + containerId + ".scope/container"); + return rootCgroupPath.resolve("machine.slice/libpod-" + containerId + ".scope/container"); } private UnixPath cpuMaxPath(ContainerId containerId) { - return new UnixPath(cgroupRoot(containerId).resolve("cpu.max")); + return new UnixPath(cgroupPath(containerId).resolve("cpu.max")); } private UnixPath cpuWeightPath(ContainerId containerId) { - return new UnixPath(cgroupRoot(containerId).resolve("cpu.weight")); + return new UnixPath(cgroupPath(containerId).resolve("cpu.weight")); } private static boolean writeCGroupsValue(NodeAgentContext context, UnixPath unixPath, String value) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java index fb789874acf..e76a46b1c3b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/Container.java @@ -51,6 +51,16 @@ public class Container extends PartialContainer { } @Override + public String toString() { + return "Container{" + + "hostname='" + hostname + '\'' + + ", resources=" + resources + + ", conmonPid=" + conmonPid + + ", networks=" + networks + + '}'; + } + + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java index f131aca2db0..ce2a6bb22ac 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java @@ -33,7 +33,7 @@ public class ContainerOperations { private final ContainerImagePruner imagePruner; private final ContainerStatsCollector containerStatsCollector; - public ContainerOperations(ContainerEngine containerEngine, CGroup cgroup, FileSystem fileSystem) { + public ContainerOperations(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem) { this.containerEngine = Objects.requireNonNull(containerEngine); this.imageDownloader = new ContainerImageDownloader(containerEngine); this.imagePruner = new ContainerImagePruner(containerEngine, Clock.systemUTC()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java index c17f98b9c9d..870809123a9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java @@ -27,15 +27,15 @@ import java.util.stream.Stream; class ContainerStatsCollector { private final ContainerEngine containerEngine; - private final CGroup cgroup; + private final CGroupV2 cgroup; private final FileSystem fileSystem; private final int onlineCpus; - ContainerStatsCollector(ContainerEngine containerEngine, CGroup cgroup, FileSystem fileSystem) { + ContainerStatsCollector(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem) { this(containerEngine, cgroup, fileSystem, Runtime.getRuntime().availableProcessors()); } - ContainerStatsCollector(ContainerEngine containerEngine, CGroup cgroup, FileSystem fileSystem, int onlineCpus) { + ContainerStatsCollector(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem, int onlineCpus) { this.containerEngine = Objects.requireNonNull(containerEngine); this.cgroup = Objects.requireNonNull(cgroup); this.fileSystem = Objects.requireNonNull(fileSystem); @@ -83,14 +83,14 @@ class ContainerStatsCollector { } private ContainerStats.CpuStats collectCpuStats(ContainerId containerId) throws IOException { - Map<CGroup.CpuStatField, Long> cpuStats = cgroup.cpuStats(containerId); + Map<CGroupV2.CpuStatField, Long> cpuStats = cgroup.cpuStats(containerId); return new ContainerStats.CpuStats(onlineCpus, systemCpuUsage(), - cpuStats.get(CGroup.CpuStatField.TOTAL_USAGE_USEC), - cpuStats.get(CGroup.CpuStatField.SYSTEM_USAGE_USEC), - cpuStats.get(CGroup.CpuStatField.THROTTLED_TIME_USEC), - cpuStats.get(CGroup.CpuStatField.TOTAL_PERIODS), - cpuStats.get(CGroup.CpuStatField.THROTTLED_PERIODS)); + cpuStats.get(CGroupV2.CpuStatField.TOTAL_USAGE_USEC), + cpuStats.get(CGroupV2.CpuStatField.SYSTEM_USAGE_USEC), + cpuStats.get(CGroupV2.CpuStatField.THROTTLED_TIME_USEC), + cpuStats.get(CGroupV2.CpuStatField.TOTAL_PERIODS), + cpuStats.get(CGroupV2.CpuStatField.THROTTLED_PERIODS)); } private ContainerStats.MemoryStats collectMemoryStats(ContainerId containerId) throws IOException { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java index 073e4263492..858b3d647fc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; +import com.yahoo.vespa.hosted.node.admin.container.ContainerId; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; /** @@ -9,6 +10,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; */ public interface ContainerWireguardTask { - void converge(NodeAgentContext context); + void converge(NodeAgentContext context, ContainerId containerId); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 45973ee6784..9e295b6a8e6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.client.zts.ZtsClientException; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.client.CsrGenerator; @@ -78,6 +79,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private final ServiceIdentityProvider hostIdentityProvider; private final IdentityDocumentClient identityDocumentClient; private final BooleanFlag tenantServiceIdentityFlag; + private final BooleanFlag useNewIdentityDocumentLayout; // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); @@ -99,6 +101,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { new AthenzIdentityVerifier(Set.of(configServerInfo.getConfigServerIdentity()))); this.clock = clock; this.tenantServiceIdentityFlag = Flags.NODE_ADMIN_TENANT_SERVICE_REGISTRY.bindTo(flagSource); + this.useNewIdentityDocumentLayout = Flags.NEW_IDDOC_LAYOUT.bindTo(flagSource); } public boolean converge(NodeAgentContext context) { @@ -134,7 +137,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { Instant now = clock.instant(); Instant expiry = certificate.getNotAfter().toInstant(); var doc = EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile); - if (doc.outdated()) { + if (refreshIdentityDocument(doc, context)) { context.log(logger, "Identity document is outdated (version=%d)", doc.documentVersion()); registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, athenzIdentity); return true; @@ -154,7 +157,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { return false; } else { lastRefreshAttempt.put(context.containerName(), now); - refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc, identityType, athenzIdentity); + refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc.identityDocument(), identityType, athenzIdentity); return true; } } @@ -165,6 +168,11 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } } + private boolean refreshIdentityDocument(SignedIdentityDocument signedIdentityDocument, NodeAgentContext context) { + int expectedVersion = documentVersion(context); + return signedIdentityDocument.outdated() || signedIdentityDocument.documentVersion() != expectedVersion; + } + public void clearCredentials(NodeAgentContext context) { FileFinder.files(context.paths().of(CONTAINER_SIA_DIRECTORY)) .deleteRecursively(context); @@ -219,7 +227,8 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private void registerIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, IdentityType identityType, AthenzIdentity identity) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - SignedIdentityDocument doc = signedIdentityDocument(context, identityType); + SignedIdentityDocument signedDoc = signedIdentityDocument(context, identityType); + IdentityDocument doc = signedDoc.identityDocument(); CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( identity, doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); @@ -231,9 +240,9 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { ztsClient.registerInstance( doc.providerService(), identity, - EntityBindingsMapper.toAttestationData(doc), + EntityBindingsMapper.toAttestationData(signedDoc), csr); - EntityBindingsMapper.writeSignedIdentityDocumentToFile(identityDocumentFile, doc); + EntityBindingsMapper.writeSignedIdentityDocumentToFile(identityDocumentFile, signedDoc); writePrivateKeyAndCertificate(privateKeyFile, keyPair.getPrivate(), certificateFile, instanceIdentity.certificate()); context.log(logger, "Instance successfully registered and credentials written to file"); } @@ -242,14 +251,14 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { /** * Return zts url from identity document, fallback to ztsEndpoint */ - private URI ztsEndpoint(SignedIdentityDocument doc) { + private URI ztsEndpoint(IdentityDocument doc) { return Optional.ofNullable(doc.ztsUrl()) .filter(s -> !s.isBlank()) .map(URI::create) .orElse(ztsEndpoint); } private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, - ContainerPath identityDocumentFile, SignedIdentityDocument doc, IdentityType identityType, AthenzIdentity identity) { + ContainerPath identityDocumentFile, IdentityDocument doc, IdentityType identityType, AthenzIdentity identity) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( @@ -310,8 +319,8 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private SignedIdentityDocument signedIdentityDocument(NodeAgentContext context, IdentityType identityType) { return switch (identityType) { - case NODE -> identityDocumentClient.getNodeIdentityDocument(context.hostname().value()); - case TENANT -> identityDocumentClient.getTenantIdentityDocument(context.hostname().value()); + case NODE -> identityDocumentClient.getNodeIdentityDocument(context.hostname().value(), documentVersion(context)); + case TENANT -> identityDocumentClient.getTenantIdentityDocument(context.hostname().value(), documentVersion(context)); }; } @@ -324,9 +333,9 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private AthenzIdentity getTenantIdentity(NodeAgentContext context, ContainerPath identityDocumentFile) { if (Files.exists(identityDocumentFile)) { - return EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile).serviceIdentity(); + return EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile).identityDocument().serviceIdentity(); } else { - return identityDocumentClient.getTenantIdentityDocument(context.hostname().value()).serviceIdentity(); + return identityDocumentClient.getTenantIdentityDocument(context.hostname().value(), documentVersion(context)).identityDocument().serviceIdentity(); } } @@ -340,6 +349,17 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { .value(); } + /* + Get the document version to ask for + */ + private int documentVersion(NodeAgentContext context) { + return useNewIdentityDocumentLayout + .with(FetchVector.Dimension.HOSTNAME, context.hostname().value()) + .value() + ? SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION + : SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION; + } + enum IdentityType { NODE("vespa-node-identity-document.json"), TENANT("vespa-tenant-identity-document.json"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index f2f690106fa..7c84afc8397 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -509,7 +509,8 @@ public class NodeAgentImpl implements NodeAgent { // TODO: this is a workaround for restarting wireguard as early as possible after host-admin has been down. var runOrdinaryWireguardTasks = true; if (container.isPresent() && container.get().state().isRunning()) { - wireguardTasks.forEach(task -> task.converge(context)); + Optional<Container> finalContainer = container; + wireguardTasks.forEach(task -> task.converge(context, finalContainer.get().id())); runOrdinaryWireguardTasks = false; } @@ -530,7 +531,10 @@ public class NodeAgentImpl implements NodeAgent { } aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); - if (runOrdinaryWireguardTasks) wireguardTasks.forEach(task -> task.converge(context)); + if (runOrdinaryWireguardTasks) { + Optional<Container> finalContainer = container; + wireguardTasks.forEach(task -> task.converge(context, finalContainer.get().id())); + } startServicesIfNeeded(context); resumeNodeIfNeeded(context); if (healthChecker.isPresent()) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1Test.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1Test.java deleted file mode 100644 index f25001d77cd..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV1Test.java +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.collections.Pair; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.test.file.TestFileSystem; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; - -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.USER_USAGE_USEC; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author freva - */ -public class CGroupV1Test { - - private static final ContainerId containerId = new ContainerId("4aec78cc"); - - private final FileSystem fileSystem = TestFileSystem.create(); - private final CGroup cgroup = new CGroupV1(fileSystem); - private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build(); - - @Test - public void updates_cpu_quota_and_period() { - assertEquals(Optional.empty(), cgroup.cpuQuotaPeriod(containerId)); - - UnixPath cpu = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/cpu/machine.slice/libpod-4aec78cc.scope")).createDirectories(); - cpu.resolve("cpu.cfs_period_us").writeUtf8File("123456\n"); - cpu.resolve("cpu.cfs_quota_us").writeUtf8File("-1\n"); - assertEquals(Optional.of(new Pair<>(-1, 123456)), cgroup.cpuQuotaPeriod(containerId)); - - cpu.resolve("cpu.cfs_quota_us").writeUtf8File("456\n"); - assertEquals(Optional.of(new Pair<>(456, 123456)), cgroup.cpuQuotaPeriod(containerId)); - - assertFalse(cgroup.updateCpuQuotaPeriod(context, containerId, 456, 123456)); - - assertTrue(cgroup.updateCpuQuotaPeriod(context, containerId, 654, 123456)); - assertEquals(Optional.of(new Pair<>(654, 123456)), cgroup.cpuQuotaPeriod(containerId)); - } - - @Test - public void updates_cpu_shares() { - assertEquals(OptionalInt.empty(), cgroup.cpuShares(containerId)); - - UnixPath cpuPath = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/cpu/machine.slice/libpod-4aec78cc.scope")).createDirectories(); - cpuPath.resolve("cpu.shares").writeUtf8File("987\n"); - assertEquals(OptionalInt.of(987), cgroup.cpuShares(containerId)); - - assertFalse(cgroup.updateCpuShares(context, containerId, 987)); - - assertTrue(cgroup.updateCpuShares(context, containerId, 789)); - assertEquals(OptionalInt.of(789), cgroup.cpuShares(containerId)); - } - - @Test - public void reads_cpu_stats() throws IOException { - UnixPath cpuacctPath = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/cpuacct/machine.slice/libpod-4aec78cc.scope")).createDirectories(); - cpuacctPath.resolve("cpuacct.usage").writeUtf8File("91623711445\n"); - cpuacctPath.resolve("cpuacct.stat").writeUtf8File("user 7463\n" + - "system 1741\n"); - cpuacctPath.resolve("cpu.stat").writeUtf8File("nr_periods 2361\n" + - "nr_throttled 342\n" + - "throttled_time 131033468519\n"); - - assertEquals(Map.of(TOTAL_USAGE_USEC, 91623711L, SYSTEM_USAGE_USEC, 17410000L, USER_USAGE_USEC, 74630000L, - TOTAL_PERIODS, 2361L, THROTTLED_PERIODS, 342L, THROTTLED_TIME_USEC, 131033468L), cgroup.cpuStats(containerId)); - } - - @Test - public void reads_memory_metrics() throws IOException { - UnixPath memoryPath = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/memory/machine.slice/libpod-4aec78cc.scope")).createDirectories(); - memoryPath.resolve("memory.usage_in_bytes").writeUtf8File("2525093888\n"); - assertEquals(2525093888L, cgroup.memoryUsageInBytes(containerId)); - - memoryPath.resolve("memory.limit_in_bytes").writeUtf8File("4322885632\n"); - assertEquals(4322885632L, cgroup.memoryLimitInBytes(containerId)); - - memoryPath.resolve("memory.stat").writeUtf8File("cache 296828928\n" + - "rss 2152587264\n" + - "rss_huge 1107296256\n" + - "shmem 135168\n" + - "mapped_file 270336\n"); - assertEquals(296828928L, cgroup.memoryCacheInBytes(containerId)); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java index 909979342ea..789f31f75c6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java @@ -14,12 +14,12 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.USER_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.USER_USAGE_USEC; import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.sharesToWeight; import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.weightToShares; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -34,7 +34,7 @@ public class CGroupV2Test { private static final ContainerId containerId = new ContainerId("4aec78cc"); private final FileSystem fileSystem = TestFileSystem.create(); - private final CGroup cgroup = new CGroupV2(fileSystem); + private final CGroupV2 cgroup = new CGroupV2(fileSystem); private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build(); private final UnixPath cgroupRoot = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-4aec78cc.scope/container")).createDirectories(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java index f852eb6235d..72c5d016a47 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java @@ -17,12 +17,12 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroup.CpuStatField.USER_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.USER_USAGE_USEC; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; @@ -37,7 +37,7 @@ public class ContainerStatsCollectorTest { private final TestTerminal testTerminal = new TestTerminal(); private final ContainerEngineMock containerEngine = new ContainerEngineMock(testTerminal); private final FileSystem fileSystem = TestFileSystem.create(); - private final CGroup cgroup = mock(CGroup.class); + private final CGroupV2 cgroup = mock(CGroupV2.class); private final NodeAgentContext context = NodeAgentContextImpl.builder(NodeSpec.Builder.testSpec("c1").build()) .fileSystem(TestFileSystem.create()) .build(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 6dc3b2b3193..fd6b15609d6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -5,6 +5,7 @@ import com.yahoo.collections.ListMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; @@ -42,6 +43,8 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.vespa.hosted.provision.restapi.NodePatcher.DROP_DOCUMENTS_REPORT; + /** * The nodes in the node repo and their state transitions * @@ -727,6 +730,23 @@ public class Nodes { return resultingNodes; } + public List<Node> dropDocuments(ApplicationId applicationId, Optional<ClusterSpec.Id> clusterId) { + try (Mutex lock = applications.lock(applicationId)) { + Instant now = clock.instant(); + List<Node> nodes = list(Node.State.active, Node.State.reserved) + .owner(applicationId) + .matching(node -> { + ClusterSpec cluster = node.allocation().get().membership().cluster(); + if (!cluster.type().isContent()) return false; + return clusterId.isEmpty() || clusterId.get().equals(cluster.id()); + }) + .mapToList(node -> node.with(node.reports().withReport(Report.basicReport(DROP_DOCUMENTS_REPORT, Report.Type.UNSPECIFIED, now, "")))); + if (nodes.isEmpty()) + throw new NoSuchNodeException("No content nodes found for " + applicationId + clusterId.map(id -> " and cluster " + id).orElse("")); + return db.writeTo(nodes, Agent.operator, Optional.empty()); + } + } + public boolean canAllocateTenantNodeTo(Node host) { return canAllocateTenantNodeTo(host, zone.cloud().dynamicProvisioning()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java index 2a4e320fd6a..401554f940d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ApplicationFilter.java @@ -36,16 +36,8 @@ public class ApplicationFilter { public static Predicate<Node> from(String applicationIds) { return makePredicate(StringUtilities.split(applicationIds).stream() - .map(ApplicationFilter::toApplicationId) + .map(ApplicationId::fromFullString) .collect(Collectors.toUnmodifiableSet())); } - public static ApplicationId toApplicationId(String applicationIdString) { - String[] parts = applicationIdString.split("\\."); - if (parts.length != 3) - throw new IllegalArgumentException("Application id must be on the form tenant.application.instance, got '" + - applicationIdString + "'"); - return ApplicationId.from(parts[0], parts[1], parts[2]); - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index bf5b735c4a0..09f947503f6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList; -import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import java.util.List; import java.util.Optional; @@ -32,7 +31,7 @@ public class LoadBalancersResponse extends SlimeJsonResponse { private Optional<ApplicationId> application() { return Optional.ofNullable(request.getProperty("application")) - .map(ApplicationFilter::toApplicationId); + .map(ApplicationId::fromFullString); } private List<LoadBalancer> loadBalancers() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index bbe287fc034..4dc48459ec9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -58,7 +58,7 @@ import static com.yahoo.config.provision.NodeResources.StorageType.remote; public class NodePatcher { // Same as in DropDocumentsReport.java - private static final String DROP_DOCUMENTS_REPORT = "dropDocuments"; + public static final String DROP_DOCUMENTS_REPORT = "dropDocuments"; private static final String WANT_TO_RETIRE = "wantToRetire"; private static final String WANT_TO_DEPROVISION = "wantToDeprovision"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index eb935ba6a5c..e04d21d3012 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostName; @@ -222,6 +223,11 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { } if (path.matches("/nodes/v2/maintenance/run/{job}")) return runJob(path.get("job")); if (path.matches("/nodes/v2/upgrade/firmware")) return requestFirmwareCheckResponse(); + if (path.matches("/nodes/v2/application/{applicationId}/drop-documents")) { + int count = nodeRepository.nodes().dropDocuments(ApplicationId.fromFullString(path.get("applicationId")), + Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)).size(); + return new MessageResponse("Triggered dropping of documents on " + count + " nodes"); + } throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); } @@ -482,14 +488,14 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { return new SlimeJsonResponse(slime); } - private void toSlime(Load load, Cursor object) { + private static void toSlime(Load load, Cursor object) { object.setDouble("cpu", load.cpu()); object.setDouble("memory", load.memory()); object.setDouble("disk", load.disk()); } /** Returns a copy of the given URI with the host and port from the given URI and the path set to the given path */ - private URI withPath(String newPath, URI uri) { + private static URI withPath(String newPath, URI uri) { try { return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), newPath, null, null); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 7affcfebdb3..022822fd3ec 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -647,22 +647,43 @@ public class NodesV2ApiTest { Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-4.json"); + } + + @Test + public void drop_documents() throws IOException { + // Initially no reports + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), "reports", false); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "reports", false); + + // Initiating drop documents will set the report on all nodes + assertResponse(new Request("http://localhost:8080/nodes/v2/application/tenant3.application3.instance3/drop-documents?clusterId=id3", new byte[0], Request.Method.POST), + "{\"message\":\"Triggered dropping of documents on 2 nodes\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), + "{\"dropDocuments\":{\"createdMillis\":123}}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), + "{\"dropDocuments\":{\"createdMillis\":123}}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com", + // Host admin of the first node finishes dropping + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"reports\": {\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36}}}"), Request.Method.PATCH), - "{\"message\":\"Updated host1.yahoo.com\"}"); - tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), + "{\"message\":\"Updated host4.yahoo.com\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "{\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36}}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host10.yahoo.com", + // Host admin of the second node finishes dropping, node-repo will update report on both nodes to start phase 2 + assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", Utf8.toBytes("{\"reports\": {\"dropDocuments\":{\"createdMillis\":49,\"droppedAt\":456}}}"), Request.Method.PATCH), - "{\"message\":\"Updated host10.yahoo.com\"}"); - tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host10.yahoo.com"), + "{\"message\":\"Updated test-node-pool-102-2\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), "{\"dropDocuments\":{\"createdMillis\":49,\"droppedAt\":456,\"readiedAt\":123}}"); - tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "{\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36,\"readiedAt\":123}}"); + + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/application/does.not.exist/drop-documents", new byte[0], Request.Method.POST), + 404, + "{\"error-code\":\"NOT_FOUND\",\"message\":\"No content nodes found for does.not.exist\"}"); } @Test diff --git a/parent/pom.xml b/parent/pom.xml index 76f4ef30dda..f3b9fc9baeb 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -212,6 +212,7 @@ <java.io.tmpdir>${project.build.directory}</java.io.tmpdir> </systemPropertyVariables> <trimStackTrace>false</trimStackTrace> + <argLine>-Djava.io.tmpdir=${project.build.directory}</argLine> </configuration> </plugin> <plugin> @@ -1154,7 +1155,7 @@ <felix.log.version>1.0.1</felix.log.version> <findbugs.version>3.0.2</findbugs.version> <!-- Should be kept in sync with guava --> <hdrhistogram.version>2.1.12</hdrhistogram.version> - <jetty.version>11.0.14</jetty.version> + <jetty.version>11.0.15</jetty.version> <jetty-servlet-api.version>5.0.2</jetty-servlet-api.version> <jjwt.version>0.11.2</jjwt.version> <jna.version>5.11.0</jna.version> diff --git a/screwdriver/update-vespa-version-in-sample-apps.sh b/screwdriver/update-vespa-version-in-sample-apps.sh index 22a55b0ce20..d3870267f26 100755 --- a/screwdriver/update-vespa-version-in-sample-apps.sh +++ b/screwdriver/update-vespa-version-in-sample-apps.sh @@ -12,8 +12,12 @@ fi readonly VESPA_RELEASE="$1" export JAVA_HOME=$(dirname $(dirname $(readlink -f /usr/bin/java))) +BUILD_DIR=$(mktemp -d) +trap "rm -rf $BUILD_DIR" EXIT +cd $BUILD_DIR + function is_published { - local TMP_MVN_REPO=/tmp/maven-repo + local TMP_MVN_REPO=$BUILD_DIR/maven-repo echo $TMP_MVN_REPO mkdir -p $TMP_MVN_REPO rm -rf $TMP_MVN_REPO/com/yahoo/vespa diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index fd6f6af730c..4ad386afa3f 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -16,7 +16,6 @@ #include <vespa/searchlib/aggregation/grouping.h> #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/attribute/extendableattributes.h> -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/engine/docsumreply.h> #include <vespa/searchlib/engine/docsumrequest.h> #include <vespa/searchlib/engine/searchreply.h> @@ -36,6 +35,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/featureset.h> #include <vespa/vespalib/util/simple_thread_bundle.h> #include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -64,6 +64,7 @@ using search::index::schema::DataType; using storage::spi::Timestamp; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; +using vespalib::FeatureSet; using vespalib::nbostream; vespalib::ThreadBundle &ttb() { return vespalib::ThreadBundle::trivial(); } diff --git a/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp b/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp index 2027ad56768..fe7303692ba 100644 --- a/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp +++ b/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp @@ -20,7 +20,6 @@ ProtonConfig make_proton_config(double concurrency, uint32_t indexing_threads = 1) { ProtonConfigBuilder builder; - // This setup requires a minimum of 4 shared threads. builder.documentdb.push_back(ProtonConfig::Documentdb()); builder.documentdb.push_back(ProtonConfig::Documentdb()); builder.flush.maxconcurrent = 1; @@ -48,8 +47,10 @@ expect_field_writer_threads(uint32_t exp_threads, uint32_t cpu_cores, uint32_t i TEST(SharedThreadingServiceConfigTest, shared_threads_are_derived_from_cpu_cores_and_feeding_concurrency) { - expect_shared_threads(4, 1); - expect_shared_threads(4, 6); + expect_shared_threads(2, 1); + expect_shared_threads(2, 4); + expect_shared_threads(3, 5); + expect_shared_threads(3, 6); expect_shared_threads(4, 8); expect_shared_threads(5, 9); expect_shared_threads(5, 10); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp index 82e1aa3b57c..430412fc1c0 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp @@ -137,8 +137,15 @@ SearchContext::getStore() const SearchContext::SearchContext(QueryTermSimple::UP qTerm, const DocumentMetaStore &toBeSearched) : search::attribute::SearchContext(toBeSearched), - _isWord(qTerm->isWord()) + _isWord(qTerm->isWord()), + _docid_limit(toBeSearched.getCommittedDocIdLimit()) { } +uint32_t +SearchContext::get_committed_docid_limit() const noexcept +{ + return _docid_limit; +} + } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h index ca4b026e2a4..7c88d8f3502 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h @@ -18,6 +18,7 @@ private: bool _isWord; document::GlobalId _gid; + uint32_t _docid_limit; unsigned int approximateHits() const override; int32_t onFind(DocId docId, int32_t elemId, int32_t &weight) const override; @@ -30,6 +31,7 @@ private: public: SearchContext(std::unique_ptr<search::QueryTermSimple> qTerm, const DocumentMetaStore &toBeSearched); + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp index fd5c3782b9a..6014df1c2f9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp @@ -18,7 +18,6 @@ #include <vespa/log/log.h> LOG_SETUP(".proton.matching.docsum_matcher"); -using search::FeatureSet; using search::MatchingElements; using search::MatchingElementsFields; using search::fef::FeatureResolver; @@ -29,6 +28,7 @@ using search::queryeval::IntermediateBlueprint; using search::queryeval::MatchingElementsSearch; using search::queryeval::SameElementBlueprint; using search::queryeval::SearchIterator; +using vespalib::FeatureSet; using AttrSearchCtx = search::attribute::ISearchContext; diff --git a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h index 006a443e539..bf99a6b1950 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h @@ -2,9 +2,9 @@ #pragma once -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> +#include <vespa/vespalib/util/featureset.h> #include <vector> #include <memory> @@ -21,7 +21,7 @@ class SearchSession; class DocsumMatcher { private: - using FeatureSet = search::FeatureSet; + using FeatureSet = vespalib::FeatureSet; using MatchingElementsFields = search::MatchingElementsFields; using MatchingElements = search::MatchingElements; diff --git a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp index 8f7970f5717..30958214b72 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp @@ -12,10 +12,10 @@ #include <vespa/searchlib/queryeval/searchiterator.h> using vespalib::Doom; +using vespalib::FeatureSet; +using vespalib::FeatureValues; using vespalib::Runnable; using vespalib::ThreadBundle; -using search::FeatureSet; -using search::FeatureValues; using search::fef::FeatureResolver; using search::fef::RankProgram; using search::queryeval::SearchIterator; diff --git a/searchcore/src/vespa/searchcore/proton/matching/extract_features.h b/searchcore/src/vespa/searchcore/proton/matching/extract_features.h index 48c3476f164..09da89250a2 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/extract_features.h +++ b/searchcore/src/vespa/searchcore/proton/matching/extract_features.h @@ -2,8 +2,8 @@ #pragma once -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/common/stringmap.h> +#include <vespa/vespalib/util/featureset.h> #include <vector> namespace vespalib { class Doom; }; @@ -16,8 +16,8 @@ namespace proton::matching { class MatchToolsFactory; struct ExtractFeatures { - using FeatureSet = search::FeatureSet; - using FeatureValues = search::FeatureValues; + using FeatureSet = vespalib::FeatureSet; + using FeatureValues = vespalib::FeatureValues; using ThreadBundle = vespalib::ThreadBundle; using SearchIterator = search::queryeval::SearchIterator; using RankProgram = search::fef::RankProgram; diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp index 3a43e9a118e..0bb183d1dc0 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp @@ -18,7 +18,7 @@ namespace proton::matching { using namespace search::fef; using search::queryeval::SearchIterator; -using search::FeatureSet; +using vespalib::FeatureSet; using vespalib::ThreadBundle; using vespalib::Issue; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 236964c2e6b..b393558638d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -27,7 +27,6 @@ using namespace search::engine; using namespace search::grouping; using search::DocumentMetaData; using search::LidUsageStats; -using search::FeatureSet; using search::MatchingElementsFields; using search::MatchingElements; using search::attribute::IAttributeContext; @@ -39,6 +38,7 @@ using search::fef::indexproperties::hitcollector::ArraySize; using search::queryeval::Blueprint; using search::queryeval::SearchIterator; using vespalib::Doom; +using vespalib::FeatureSet; using vespalib::make_string_short::fmt; namespace proton::matching { diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.h b/searchcore/src/vespa/searchcore/proton/matching/matcher.h index bad56fe1c36..6507ffca2eb 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.h @@ -11,13 +11,13 @@ #include "viewresolver.h" #include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchlib/fef/blueprintfactory.h> -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/resultset.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/query/base.h> #include <vespa/vespalib/util/clock.h> +#include <vespa/vespalib/util/featureset.h> #include <vespa/vespalib/util/thread_bundle.h> #include <mutex> @@ -135,7 +135,7 @@ public: * @param attrCtx abstract view of attribute data * @return calculated summary features. **/ - search::FeatureSet::SP + vespalib::FeatureSet::SP getSummaryFeatures(const DocsumRequest & req, ISearchContext & searchCtx, IAttributeContext & attrCtx, SessionManager &sessionManager) const; @@ -149,7 +149,7 @@ public: * @param attrCtx abstract view of attribute data * @return calculated rank features. **/ - search::FeatureSet::SP + vespalib::FeatureSet::SP getRankFeatures(const DocsumRequest & req, ISearchContext & searchCtx, IAttributeContext & attrCtx, SessionManager &sessionManager) const; diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp index 50ef5039e75..c1802b40deb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp @@ -28,10 +28,10 @@ namespace { uint32_t derive_shared_threads(const ProtonConfig& cfg, const HwInfo::Cpu& cpu_info) { - uint32_t scaled_cores = (uint32_t)std::ceil(cpu_info.cores() * cfg.feeding.concurrency); + uint32_t scaled_cores = uint32_t(std::ceil(cpu_info.cores() * cfg.feeding.concurrency)); // We need at least 1 guaranteed free worker in order to ensure progress. - return std::max(scaled_cores, (uint32_t)cfg.documentdb.size() + cfg.flush.maxconcurrent + 1); + return std::max(scaled_cores, uint32_t(cfg.flush.maxconcurrent + 1u)); } uint32_t @@ -42,8 +42,8 @@ derive_warmup_threads(const HwInfo::Cpu& cpu_info) { uint32_t derive_field_writer_threads(const ProtonConfig& cfg, const HwInfo::Cpu& cpu_info) { - uint32_t scaled_cores = (size_t)std::ceil(cpu_info.cores() * cfg.feeding.concurrency); - uint32_t field_writer_threads = std::max(scaled_cores, (uint32_t)cfg.indexing.threads); + uint32_t scaled_cores = size_t(std::ceil(cpu_info.cores() * cfg.feeding.concurrency)); + uint32_t field_writer_threads = std::max(scaled_cores, uint32_t(cfg.indexing.threads)); // Originally we used at least 3 threads for writing fields: // - index field inverter // - index field writer diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp index 9e65dfcfc07..e55344aded0 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp +++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp @@ -240,6 +240,24 @@ TEST_F("original lid range is used by read guard", Fixture) EXPECT_EQUAL(getUndefined<int>(), first_guard->getInt(DocId(10))); } +TEST_F("Original target lid range is used by read guard", Fixture) +{ + reset_with_single_value_reference_mappings<IntegerAttribute, int32_t>( + f, BasicType::INT32, + {}); + EXPECT_EQUAL(11u, f.target_attr->getNumDocs()); + auto first_guard = f.get_imported_attr(); + add_n_docs_with_undefined_values(*f.target_attr, 1); + EXPECT_EQUAL(12u, f.target_attr->getNumDocs()); + auto typed_target_attr = f.template target_attr_as<IntegerAttribute>(); + ASSERT_TRUE(typed_target_attr->update(11, 2345)); + f.target_attr->commit(); + f.map_reference(DocId(8), dummy_gid(11), DocId(11)); + auto second_guard = f.get_imported_attr(); + EXPECT_EQUAL(2345, second_guard->getInt(DocId(8))); + EXPECT_NOT_EQUAL(2345, first_guard->getInt(DocId(8))); +} + struct SingleStringAttrFixture : Fixture { SingleStringAttrFixture() : Fixture() { setup(); diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp index 847a992d241..19327245083 100644 --- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp +++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp @@ -429,6 +429,21 @@ TEST_F("original lid range is used by search context", SingleValueFixture) EXPECT_TRUE(second_ctx->matches(DocId(10))); } +TEST_F("Original target lid range is used by search context", SingleValueFixture) +{ + EXPECT_EQUAL(11u, f.target_attr->getNumDocs()); + auto first_ctx = f.create_context(word_term("2345")); + add_n_docs_with_undefined_values(*f.target_attr, 1); + EXPECT_EQUAL(12u, f.target_attr->getNumDocs()); + auto typed_target_attr = f.template target_attr_as<IntegerAttribute>(); + ASSERT_TRUE(typed_target_attr->update(11, 2345)); + f.target_attr->commit(); + f.map_reference(DocId(8), dummy_gid(11), DocId(11)); + auto second_ctx = f.create_context(word_term("2345")); + EXPECT_FALSE(first_ctx->matches(DocId(8))); + EXPECT_TRUE(second_ctx->matches(DocId(8))); +} + // Note: this uses an underlying string attribute, as queryTerm() does not seem to // implemented at all for (single) numeric attributes. Intentional? TEST_F("queryTerm() returns term context was created with", WsetValueFixture) { diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index 2804e3f74e4..5ba90d2b077 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -390,7 +390,7 @@ TEST("testSingleValue") { EXPECT_EQUAL(24u, sizeof(SearchContext)); EXPECT_EQUAL(32u, sizeof(StringSearchHelper)); - EXPECT_EQUAL(80u, sizeof(attribute::SingleStringEnumSearchContext)); + EXPECT_EQUAL(88u, sizeof(attribute::SingleStringEnumSearchContext)); { Config cfg(BasicType::STRING, CollectionType::SINGLE); SingleValueStringAttribute svsa("svsa", cfg); diff --git a/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp b/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp index 834cdbef50d..73a81be9f90 100644 --- a/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp +++ b/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp @@ -2,9 +2,9 @@ #include <vespa/log/log.h> LOG_SETUP("summaryfeatures_test"); #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/common/featureset.h> +#include <vespa/vespalib/util/featureset.h> -using namespace search; +using vespalib::FeatureSet; using vespalib::Memory; TEST_SETUP(Test); diff --git a/searchlib/src/tests/query/streaming_query_test.cpp b/searchlib/src/tests/query/streaming_query_test.cpp index 2c202d9131b..210f32af15e 100644 --- a/searchlib/src/tests/query/streaming_query_test.cpp +++ b/searchlib/src/tests/query/streaming_query_test.cpp @@ -814,7 +814,7 @@ TEST("test_nearest_neighbor_query_node") constexpr uint32_t target_num_hits = 100; constexpr bool allow_approximate = false; constexpr uint32_t explore_additional_hits = 800; - constexpr double raw_score = 0.5; + constexpr double distance = 0.5; builder.add_nearest_neighbor_term("qtensor", "field", id, Weight(weight), target_num_hits, allow_approximate, explore_additional_hits, distance_threshold); auto build_node = builder.build(); auto stack_dump = StackDumpCreator::create(*build_node); @@ -830,14 +830,14 @@ TEST("test_nearest_neighbor_query_node") EXPECT_EQUAL(id, static_cast<int32_t>(node->uniqueId())); EXPECT_EQUAL(weight, node->weight().percent()); EXPECT_EQUAL(distance_threshold, node->get_distance_threshold()); - EXPECT_FALSE(node->get_raw_score().has_value()); + EXPECT_FALSE(node->get_distance().has_value()); EXPECT_FALSE(node->evaluate()); - node->set_raw_score(raw_score); - EXPECT_TRUE(node->get_raw_score().has_value()); - EXPECT_EQUAL(raw_score, node->get_raw_score().value()); + node->set_distance(distance); + EXPECT_TRUE(node->get_distance().has_value()); + EXPECT_EQUAL(distance, node->get_distance().value()); EXPECT_TRUE(node->evaluate()); node->reset(); - EXPECT_FALSE(node->get_raw_score().has_value()); + EXPECT_FALSE(node->get_distance().has_value()); EXPECT_FALSE(node->evaluate()); } diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index 86b83b2c651..e5bed3ebae5 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -44,6 +44,32 @@ void verify_geo_miles(const DistanceFunction *dist_fun, } } +double computeEuclideanChecked(TypedCells a, TypedCells b) { + static EuclideanDistanceFunctionFactory<Int8Float> i8f_dff; + static EuclideanDistanceFunctionFactory<float> flt_dff; + static EuclideanDistanceFunctionFactory<double> dbl_dff; + auto d_n = dbl_dff.for_query_vector(a); + auto d_f = flt_dff.for_query_vector(a); + auto d_r = dbl_dff.for_query_vector(b); + auto d_i = dbl_dff.for_insertion_vector(a); + // normal: + double result = d_n->calc(b); + // insert is exactly same: + EXPECT_EQ(d_i->calc(b), result); + // reverse: + EXPECT_DOUBLE_EQ(d_r->calc(a), result); + // float factory: + EXPECT_FLOAT_EQ(d_f->calc(b), result); + if (a.type == vespalib::eval::CellType::INT8 || + b.type == vespalib::eval::CellType::INT8) + { + auto d_8 = i8f_dff.for_query_vector(a); + EXPECT_DOUBLE_EQ(d_8->calc(b), result); + } + return result; +} + +namespace { constexpr double sq_root_half = std::sqrt(0.5); } TEST(DistanceFunctionsTest, euclidean_gives_expected_score) { @@ -55,19 +81,60 @@ TEST(DistanceFunctionsTest, euclidean_gives_expected_score) std::vector<double> p1{1.0, 0.0, 0.0}; std::vector<double> p2{0.0, 1.0, 0.0}; std::vector<double> p3{0.0, 0.0, 1.0}; - std::vector<double> p4{0.5, 0.5, 0.707107}; + std::vector<double> p4{0.5, 0.5, sq_root_half}; std::vector<double> p5{0.0,-1.0, 0.0}; std::vector<double> p6{1.0, 2.0, 2.0}; - double n4 = euclid->calc(t(p0), t(p4)); + double n4 = computeEuclideanChecked(t(p0), t(p4)); EXPECT_FLOAT_EQ(n4, 1.0); - double d12 = euclid->calc(t(p1), t(p2)); + double d12 = computeEuclideanChecked(t(p1), t(p2)); EXPECT_EQ(d12, 2.0); EXPECT_DOUBLE_EQ(euclid->to_rawscore(d12), 1.0/(1.0 + sqrt(2.0))); double threshold = euclid->convert_threshold(8.0); EXPECT_EQ(threshold, 64.0); threshold = euclid->convert_threshold(0.5); EXPECT_EQ(threshold, 0.25); + + // simple hand-checked distances: + EXPECT_EQ(computeEuclideanChecked(t(p0), t(p0)), 0.0); + EXPECT_EQ(computeEuclideanChecked(t(p0), t(p1)), 1.0); + EXPECT_EQ(computeEuclideanChecked(t(p0), t(p2)), 1.0); + EXPECT_EQ(computeEuclideanChecked(t(p0), t(p3)), 1.0); + EXPECT_EQ(computeEuclideanChecked(t(p0), t(p5)), 1.0); + EXPECT_EQ(computeEuclideanChecked(t(p0), t(p6)), 9.0); + + EXPECT_EQ(computeEuclideanChecked(t(p1), t(p1)), 0.0); + EXPECT_EQ(computeEuclideanChecked(t(p1), t(p2)), 2.0); + EXPECT_EQ(computeEuclideanChecked(t(p1), t(p3)), 2.0); + EXPECT_EQ(computeEuclideanChecked(t(p1), t(p5)), 2.0); + EXPECT_EQ(computeEuclideanChecked(t(p1), t(p6)), 8.0); + + EXPECT_EQ(computeEuclideanChecked(t(p2), t(p2)), 0.0); + EXPECT_EQ(computeEuclideanChecked(t(p2), t(p3)), 2.0); + EXPECT_EQ(computeEuclideanChecked(t(p2), t(p5)), 4.0); + EXPECT_EQ(computeEuclideanChecked(t(p2), t(p6)), 6.0); + + EXPECT_EQ(computeEuclideanChecked(t(p3), t(p3)), 0.0); + EXPECT_EQ(computeEuclideanChecked(t(p3), t(p5)), 2.0); + EXPECT_EQ(computeEuclideanChecked(t(p3), t(p6)), 6.0); + + EXPECT_EQ(computeEuclideanChecked(t(p5), t(p5)), 0.0); + EXPECT_EQ(computeEuclideanChecked(t(p5), t(p6)), 14.0); + + EXPECT_EQ(computeEuclideanChecked(t(p6), t(p6)), 0.0); + + // smoke test for bfloat16: + std::vector<vespalib::BFloat16> bf16v; + bf16v.emplace_back(1.0); + bf16v.emplace_back(1.0); + bf16v.emplace_back(1.0); + EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p0)), 3.0); + EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p1)), 2.0); + EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p2)), 2.0); + EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p3)), 2.0); + EXPECT_FLOAT_EQ(computeEuclideanChecked(t(bf16v), t(p4)), 0.5857863); + EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p5)), 6.0); + EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p6)), 2.0); } TEST(DistanceFunctionsTest, euclidean_int8_smoketest) @@ -81,14 +148,13 @@ TEST(DistanceFunctionsTest, euclidean_int8_smoketest) std::vector<Int8Float> p5{0.0,-1.0, 0.0}; std::vector<Int8Float> p7{-1.0, 2.0, -2.0}; - EXPECT_DOUBLE_EQ(1.0, euclid->calc(t(p0), t(p1))); - EXPECT_DOUBLE_EQ(1.0, euclid->calc(t(p0), t(p5))); - EXPECT_DOUBLE_EQ(9.0, euclid->calc(t(p0), t(p7))); - - EXPECT_DOUBLE_EQ(2.0, euclid->calc(t(p1), t(p5))); - EXPECT_DOUBLE_EQ(12.0, euclid->calc(t(p1), t(p7))); - EXPECT_DOUBLE_EQ(14.0, euclid->calc(t(p5), t(p7))); + EXPECT_DOUBLE_EQ(1.0, computeEuclideanChecked(t(p0), t(p1))); + EXPECT_DOUBLE_EQ(1.0, computeEuclideanChecked(t(p0), t(p5))); + EXPECT_DOUBLE_EQ(9.0, computeEuclideanChecked(t(p0), t(p7))); + EXPECT_DOUBLE_EQ(2.0, computeEuclideanChecked(t(p1), t(p5))); + EXPECT_DOUBLE_EQ(12.0, computeEuclideanChecked(t(p1), t(p7))); + EXPECT_DOUBLE_EQ(14.0, computeEuclideanChecked(t(p5), t(p7))); } double computeAngularChecked(TypedCells a, TypedCells b) { @@ -115,7 +181,7 @@ TEST(DistanceFunctionsTest, angular_gives_expected_score) std::vector<double> p1{1.0, 0.0, 0.0}; std::vector<double> p2{0.0, 1.0, 0.0}; std::vector<double> p3{0.0, 0.0, 1.0}; - std::vector<double> p4{0.5, 0.5, 0.707107}; + std::vector<double> p4{0.5, 0.5, sq_root_half}; std::vector<double> p5{0.0,-1.0, 0.0}; std::vector<double> p6{1.0, 2.0, 2.0}; @@ -143,7 +209,7 @@ TEST(DistanceFunctionsTest, angular_gives_expected_score) EXPECT_DOUBLE_EQ(threshold, 0.5); double a34 = computeAngularChecked(t(p3), t(p4)); - EXPECT_FLOAT_EQ(a34, (1.0 - 0.707107)); + EXPECT_FLOAT_EQ(a34, (1.0 - sq_root_half)); EXPECT_FLOAT_EQ(angular->to_rawscore(a34), 1.0/(1.0 + pi/4)); threshold = angular->convert_threshold(pi/4); EXPECT_FLOAT_EQ(threshold, a34); @@ -193,6 +259,89 @@ TEST(DistanceFunctionsTest, angular_gives_expected_score) EXPECT_DOUBLE_EQ(a66, computeAngularChecked(t(iv6), t(iv6))); } +double computePrenormalizedAngularChecked(TypedCells a, TypedCells b) { + static PrenormalizedAngularDistanceFunctionFactory<float> flt_dff; + static PrenormalizedAngularDistanceFunctionFactory<double> dbl_dff; + auto d_n = dbl_dff.for_query_vector(a); + auto d_f = flt_dff.for_query_vector(a); + auto d_r = dbl_dff.for_query_vector(b); + auto d_i = dbl_dff.for_insertion_vector(a); + // normal: + double result = d_n->calc(b); + // insert is exactly same: + EXPECT_EQ(d_i->calc(b), result); + // note: for this distance, reverse is not necessarily equal, + // since we normalize based on length of LHS only + EXPECT_FLOAT_EQ(d_r->calc(a), result); + // float factory: + EXPECT_FLOAT_EQ(d_f->calc(b), result); + double closeness_n = d_n->to_rawscore(result); + double closeness_f = d_f->to_rawscore(result); + double closeness_r = d_r->to_rawscore(result); + double closeness_i = d_i->to_rawscore(result); + EXPECT_DOUBLE_EQ(closeness_n, closeness_f); + EXPECT_DOUBLE_EQ(closeness_n, closeness_r); + EXPECT_DOUBLE_EQ(closeness_n, closeness_i); + EXPECT_GT(closeness_n, 0.0); + EXPECT_LE(closeness_n, 1.0); + return result; +} + +TEST(DistanceFunctionsTest, prenormalized_angular_gives_expected_score) +{ + std::vector<double> p0{0.0, 0.0, 0.0}; + std::vector<double> p1{1.0, 0.0, 0.0}; + std::vector<double> p2{0.0, 1.0, 0.0}; + std::vector<double> p3{0.0, 0.0, 1.0}; + std::vector<double> p4{0.5, 0.5, sq_root_half}; + std::vector<double> p5{0.0,-1.0, 0.0}; + std::vector<double> p6{1.0, 2.0, 2.0}; + std::vector<double> p7{2.0, -1.0, -2.0}; + std::vector<double> p8{3.0, 0.0, 0.0}; + + PrenormalizedAngularDistanceFunctionFactory<double> dff; + auto pnad = dff.for_query_vector(t(p0)); + + double i12 = computePrenormalizedAngularChecked(t(p1), t(p2)); + double i13 = computePrenormalizedAngularChecked(t(p1), t(p3)); + double i23 = computePrenormalizedAngularChecked(t(p2), t(p3)); + EXPECT_DOUBLE_EQ(i12, 1.0); + EXPECT_DOUBLE_EQ(i13, 1.0); + EXPECT_DOUBLE_EQ(i23, 1.0); + + double i14 = computePrenormalizedAngularChecked(t(p1), t(p4)); + double i24 = computePrenormalizedAngularChecked(t(p2), t(p4)); + EXPECT_DOUBLE_EQ(i14, 0.5); + EXPECT_DOUBLE_EQ(i24, 0.5); + double i34 = computePrenormalizedAngularChecked(t(p3), t(p4)); + EXPECT_FLOAT_EQ(i34, 1.0 - sq_root_half); + + double i25 = computePrenormalizedAngularChecked(t(p2), t(p5)); + EXPECT_DOUBLE_EQ(i25, 2.0); + + double i44 = computePrenormalizedAngularChecked(t(p4), t(p4)); + EXPECT_GE(i44, 0.0); + EXPECT_LT(i44, 0.000001); + + double i66 = computePrenormalizedAngularChecked(t(p6), t(p6)); + EXPECT_GE(i66, 0.0); + EXPECT_LT(i66, 0.000001); + + double i67 = computePrenormalizedAngularChecked(t(p6), t(p7)); + EXPECT_DOUBLE_EQ(i67, 13.0); + double i68 = computePrenormalizedAngularChecked(t(p6), t(p8)); + EXPECT_DOUBLE_EQ(i68, 6.0); + double i78 = computePrenormalizedAngularChecked(t(p7), t(p8)); + EXPECT_DOUBLE_EQ(i78, 3.0); + + double threshold = pnad->convert_threshold(0.25); + EXPECT_DOUBLE_EQ(threshold, 0.25); + threshold = pnad->convert_threshold(0.5); + EXPECT_DOUBLE_EQ(threshold, 0.5); + threshold = pnad->convert_threshold(1.0); + EXPECT_DOUBLE_EQ(threshold, 1.0); +} + TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) { auto ct = vespalib::eval::CellType::DOUBLE; @@ -203,7 +352,7 @@ TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) std::vector<double> p1{1.0, 0.0, 0.0}; std::vector<double> p2{0.0, 1.0, 0.0}; std::vector<double> p3{0.0, 0.0, 1.0}; - std::vector<double> p4{0.5, 0.5, 0.707107}; + std::vector<double> p4{0.5, 0.5, sq_root_half}; std::vector<double> p5{0.0,-1.0, 0.0}; std::vector<double> p6{1.0, 2.0, 2.0}; @@ -219,7 +368,7 @@ TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) EXPECT_DOUBLE_EQ(i14, 0.5); EXPECT_DOUBLE_EQ(i24, 0.5); double i34 = innerproduct->calc(t(p3), t(p4)); - EXPECT_FLOAT_EQ(i34, 1.0 - 0.707107); + EXPECT_FLOAT_EQ(i34, 1.0 - sq_root_half); double i25 = innerproduct->calc(t(p2), t(p5)); EXPECT_DOUBLE_EQ(i25, 2.0); @@ -228,6 +377,10 @@ TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) EXPECT_GE(i44, 0.0); EXPECT_LT(i44, 0.000001); + double i66 = innerproduct->calc(t(p6), t(p6)); + EXPECT_GE(i66, 0.0); + EXPECT_LT(i66, 0.000001); + double threshold = innerproduct->convert_threshold(0.25); EXPECT_DOUBLE_EQ(threshold, 0.25); threshold = innerproduct->convert_threshold(0.5); diff --git a/searchlib/src/vespa/searchcommon/attribute/i_search_context.h b/searchlib/src/vespa/searchcommon/attribute/i_search_context.h index 8867d1b87e4..4657d41a4a0 100644 --- a/searchlib/src/vespa/searchcommon/attribute/i_search_context.h +++ b/searchlib/src/vespa/searchcommon/attribute/i_search_context.h @@ -70,6 +70,11 @@ public: bool matches(DocId docId, int32_t &weight) const { return matches(*this, docId, weight); } bool matches(DocId doc) const { return find(doc, 0) >= 0; } + /* + * Committed docid limit on attribute vector when search context was + * created. + */ + virtual uint32_t get_committed_docid_limit() const noexcept = 0; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/empty_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/empty_search_context.cpp index 7a5d82cd9ba..91bdb45ff19 100644 --- a/searchlib/src/vespa/searchlib/attribute/empty_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/empty_search_context.cpp @@ -30,6 +30,12 @@ EmptySearchContext::approximateHits() const return 0u; } +uint32_t +EmptySearchContext::get_committed_docid_limit() const noexcept +{ + return 0u; +} + std::unique_ptr<queryeval::SearchIterator> EmptySearchContext::createIterator(fef::TermFieldMatchData*, bool) { diff --git a/searchlib/src/vespa/searchlib/attribute/empty_search_context.h b/searchlib/src/vespa/searchlib/attribute/empty_search_context.h index ae6f6d76edf..133e540d87f 100644 --- a/searchlib/src/vespa/searchlib/attribute/empty_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/empty_search_context.h @@ -19,6 +19,7 @@ class EmptySearchContext : public SearchContext public: EmptySearchContext(const AttributeVector& attr) noexcept; ~EmptySearchContext(); + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h index 0342976ffd6..86ffa1c8ab0 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h @@ -41,6 +41,7 @@ protected: void fetchPostings(const queryeval::ExecuteInfo & execInfo) override; unsigned int approximateHits() const override; + uint32_t get_committed_docid_limit() const noexcept { return _docIdLimit; } }; } diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h index cc2cb216bbb..bc884f9020e 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h @@ -2,7 +2,7 @@ /** * @class search::SearchVisitor * - * @brief Visitor that applies a search query to visitor data and converts them to a SearchResultCommand + * @brief Visitor that applies a search query to visitor data and converts them to a QueryResultCommand */ #pragma once diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.hpp b/searchlib/src/vespa/searchlib/attribute/extendableattributes.hpp index c515a1bec7f..0b1d1221df2 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.hpp +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.hpp @@ -2,7 +2,7 @@ /** * @class search::SearchVisitor * - * @brief Visitor that applies a search query to visitor data and converts them to a SearchResultCommand + * @brief Visitor that applies a search query to visitor data and converts them to a QueryResultCommand */ #pragma once diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index a1a5e9f7894..b50a3720ff8 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -17,12 +17,14 @@ ImportedAttributeVectorReadGuard::ImportedAttributeVectorReadGuard(std::shared_p _target_document_meta_store_read_guard(std::move(targetMetaStoreReadGuard)), _imported_attribute(imported_attribute), _targetLids(), + _target_docid_limit(0u), _reference_attribute_guard(imported_attribute.getReferenceAttribute()), _target_attribute_guard(imported_attribute.getTargetAttribute()->makeReadGuard(stableEnumGuard)), _reference_attribute(*imported_attribute.getReferenceAttribute()), _target_attribute(*_target_attribute_guard->attribute()) { _targetLids = _reference_attribute.getTargetLids(); + _target_docid_limit = _target_attribute.getCommittedDocIdLimit(); } ImportedAttributeVectorReadGuard::~ImportedAttributeVectorReadGuard() = default; diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index cb48399f688..1297acad9b8 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -95,6 +95,7 @@ private: std::shared_ptr<MetaStoreReadGuard> _target_document_meta_store_read_guard; const ImportedAttributeVector &_imported_attribute; TargetLids _targetLids; + uint32_t _target_docid_limit; AttributeGuard _reference_attribute_guard; std::unique_ptr<attribute::AttributeReadGuard> _target_attribute_guard; const ReferenceAttribute &_reference_attribute; @@ -103,7 +104,9 @@ protected: uint32_t getTargetLid(uint32_t lid) const { // Check range to avoid reading memory beyond end of mapping array - return lid < _targetLids.size() ? _targetLids[lid].load_acquire() : 0u; + uint32_t target_lid = lid < _targetLids.size() ? _targetLids[lid].load_acquire() : 0u; + // Check target range + return target_lid < _target_docid_limit ? target_lid : 0u; } long onSerializeForAscendingSort(DocId doc, void * serTo, long available, diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index 1e8adc3922e..3d308b82b04 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -43,6 +43,7 @@ ImportedSearchContext::ImportedSearchContext( _target_attribute(target_attribute), _target_search_context(_target_attribute.createSearchContext(std::move(term), params)), _targetLids(_reference_attribute.getTargetLids()), + _target_docid_limit(_target_search_context->get_committed_docid_limit()), _merger(_reference_attribute.getCommittedDocIdLimit()), _params(params), _zero_hits(false) @@ -327,4 +328,10 @@ const vespalib::string& ImportedSearchContext::attributeName() const { return _imported_attribute.getName(); } +uint32_t +ImportedSearchContext::get_committed_docid_limit() const noexcept +{ + return _targetLids.size(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index d9c09d8c645..d6b6d09e8fc 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -39,6 +39,7 @@ class ImportedSearchContext : public ISearchContext { const IAttributeVector &_target_attribute; std::unique_ptr<ISearchContext> _target_search_context; TargetLids _targetLids; + uint32_t _target_docid_limit; PostingListMerger<int32_t> _merger; SearchContextParams _params; mutable std::atomic<bool> _zero_hits; @@ -47,7 +48,9 @@ class ImportedSearchContext : public ISearchContext { uint32_t getTargetLid(uint32_t lid) const { // Check range to avoid reading memory beyond end of mapping array - return lid < _targetLids.size() ? _targetLids[lid].load_acquire() : 0u; + uint32_t target_lid = lid < _targetLids.size() ? _targetLids[lid].load_acquire() : 0u; + // Check target range + return target_lid < _target_docid_limit ? target_lid : 0u; } void makeMergedPostings(bool isFilter); @@ -90,6 +93,7 @@ public: const ISearchContext &target_search_context() const noexcept { return *_target_search_context; } + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h index 5b393d8bdb2..161c6799787 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h @@ -59,6 +59,7 @@ public: std::unique_ptr<queryeval::SearchIterator> createFilterIterator(fef::TermFieldMatchData* matchData, bool strict) override; + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp index e7901199e50..15abcf6f0d9 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp @@ -33,4 +33,11 @@ MultiEnumSearchContext<T, BaseSC, M>::createFilterIterator(fef::TermFieldMatchDa : std::make_unique<AttributeIteratorT<MultiEnumSearchContext>>(*this, matchData); } +template <typename T, typename BaseSC, typename M> +uint32_t +MultiEnumSearchContext<T, BaseSC, M>::get_committed_docid_limit() const noexcept +{ + return _mv_mapping_read_view.get_committed_docid_limit(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h index b2c76a120f9..23e56e23af9 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h @@ -54,6 +54,7 @@ public: std::unique_ptr<queryeval::SearchIterator> createFilterIterator(fef::TermFieldMatchData* matchData, bool strict) override; + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp index 15b851215f8..7e1fd1aeb5a 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp @@ -33,4 +33,11 @@ MultiNumericSearchContext<T, M>::createFilterIterator(fef::TermFieldMatchData* m : std::make_unique<AttributeIteratorT<MultiNumericSearchContext<T, M>>>(*this, matchData); } +template <typename T, typename M> +uint32_t +MultiNumericSearchContext<T, M>::get_committed_docid_limit() const noexcept +{ + return _mv_mapping_read_view.get_committed_docid_limit(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h index 41138ff0890..609989208c3 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h @@ -33,6 +33,7 @@ public: } vespalib::ConstArrayRef<ElemT> get(uint32_t doc_id) const { return _store->get(_indices[doc_id].load_acquire()); } bool valid() const noexcept { return _store != nullptr; } + uint32_t get_committed_docid_limit() const noexcept { return _indices.size(); } }; } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index b701a6fd08f..9343dafe917 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -454,12 +454,14 @@ class ReferenceSearchContext : public attribute::SearchContext { private: const ReferenceAttribute& _ref_attr; GlobalId _term; + uint32_t _docid_limit; public: ReferenceSearchContext(const ReferenceAttribute& ref_attr, const GlobalId& term) : attribute::SearchContext(ref_attr), _ref_attr(ref_attr), - _term(term) + _term(term), + _docid_limit(ref_attr.getCommittedDocIdLimit()) { } bool valid() const override { @@ -480,8 +482,15 @@ public: int32_t weight; return onFind(docId, elementId, weight); } + uint32_t get_committed_docid_limit() const noexcept override; }; +uint32_t +ReferenceSearchContext::get_committed_docid_limit() const noexcept +{ + return _docid_limit; +} + } std::unique_ptr<attribute::SearchContext> diff --git a/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.h index 83d6c696117..f6a2f94dedb 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.h @@ -17,7 +17,9 @@ class SingleEnumSearchContext : public BaseSC { protected: using DocId = ISearchContext::DocId; - const vespalib::datastore::AtomicEntryRef* _enum_indices; + using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; + using EnumIndices = vespalib::ConstArrayRef<AtomicEntryRef>; + EnumIndices _enum_indices; const EnumStoreT<T>& _enum_store; int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const final { @@ -29,7 +31,7 @@ protected: } public: - SingleEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<T>& enum_store); + SingleEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<T>& enum_store); int32_t find(DocId docId, int32_t elemId, int32_t & weight) const { if ( elemId != 0) return -1; @@ -46,6 +48,7 @@ public: std::unique_ptr<queryeval::SearchIterator> createFilterIterator(fef::TermFieldMatchData* matchData, bool strict) override; + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.hpp index a415c301f9c..6b6cf480d6a 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.hpp @@ -9,7 +9,7 @@ namespace search::attribute { template <typename T, typename BaseSC> -SingleEnumSearchContext<T, BaseSC>::SingleEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<T>& enum_store) +SingleEnumSearchContext<T, BaseSC>::SingleEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<T>& enum_store) : BaseSC(toBeSearched, std::move(matcher)), _enum_indices(enum_indices), _enum_store(enum_store) @@ -33,4 +33,11 @@ SingleEnumSearchContext<T, BaseSC>::createFilterIterator(fef::TermFieldMatchData : std::make_unique<AttributeIteratorT<SingleEnumSearchContext>>(*this, matchData); } +template <typename T, typename BaseSC> +uint32_t +SingleEnumSearchContext<T, BaseSC>::get_committed_docid_limit() const noexcept +{ + return _enum_indices.size(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.h index 86283f59283..fd3f4c03a8a 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.h @@ -16,7 +16,9 @@ template <typename T> class SingleNumericEnumSearchContext : public SingleEnumSearchContext<T, NumericSearchContext<NumericRangeMatcher<T>>> { public: - SingleNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<T>& enum_store); + using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; + using EnumIndices = vespalib::ConstArrayRef<AtomicEntryRef>; + SingleNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<T>& enum_store); }; } diff --git a/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.hpp index f4e049cb6f1..c0818d4d18a 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.hpp @@ -8,7 +8,7 @@ namespace search::attribute { template <typename T> -SingleNumericEnumSearchContext<T>::SingleNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<T>& enum_store) +SingleNumericEnumSearchContext<T>::SingleNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<T>& enum_store) : SingleEnumSearchContext<T, NumericSearchContext<NumericRangeMatcher<T>>>(NumericRangeMatcher<T>(*qTerm, true), toBeSearched, enum_indices, enum_store) { } diff --git a/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.h index 5f6925f7f4d..6362c69cdac 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.h @@ -3,6 +3,7 @@ #pragma once #include "numeric_search_context.h" +#include <vespa/vespalib/util/arrayref.h> #include <vespa/vespalib/util/atomic.h> namespace search::attribute { @@ -16,7 +17,7 @@ class SingleNumericSearchContext final : public NumericSearchContext<M> { private: using DocId = ISearchContext::DocId; - const T* _data; + vespalib::ConstArrayRef<T> _data; int32_t onFind(DocId docId, int32_t elemId, int32_t& weight) const override { return find(docId, elemId, weight); @@ -27,7 +28,7 @@ private: } public: - SingleNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const T* data); + SingleNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, vespalib::ConstArrayRef<T> data); int32_t find(DocId docId, int32_t elemId, int32_t& weight) const { if ( elemId != 0) return -1; const T v = vespalib::atomic::load_ref_relaxed(_data[docId]); @@ -43,6 +44,7 @@ public: std::unique_ptr<queryeval::SearchIterator> createFilterIterator(fef::TermFieldMatchData* matchData, bool strict) override; + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.hpp index 75d3da9de7f..b40b1336e6f 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.hpp @@ -9,7 +9,7 @@ namespace search::attribute { template <typename T, typename M> -SingleNumericSearchContext<T, M>::SingleNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const T* data) +SingleNumericSearchContext<T, M>::SingleNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, vespalib::ConstArrayRef<T> data) : NumericSearchContext<M>(toBeSearched, *qTerm, true), _data(data) { @@ -32,4 +32,11 @@ SingleNumericSearchContext<T, M>::createFilterIterator(fef::TermFieldMatchData* : std::make_unique<AttributeIteratorT<SingleNumericSearchContext<T, M>>>(*this, matchData); } +template <typename T, typename M> +uint32_t +SingleNumericSearchContext<T, M>::get_committed_docid_limit() const noexcept +{ + return _data.size(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.cpp index 5eeef7cd61a..074435809cc 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.cpp @@ -6,13 +6,14 @@ namespace search::attribute { -SingleSmallNumericSearchContext::SingleSmallNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const Word* word_data, Word value_mask, uint32_t value_shift_shift, uint32_t value_shift_mask, uint32_t word_shift) +SingleSmallNumericSearchContext::SingleSmallNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const Word* word_data, Word value_mask, uint32_t value_shift_shift, uint32_t value_shift_mask, uint32_t word_shift, uint32_t docid_limit) : NumericSearchContext<NumericRangeMatcher<T>>(toBeSearched, *qTerm, false), _wordData(word_data), _valueMask(value_mask), _valueShiftShift(value_shift_shift), _valueShiftMask(value_shift_mask), - _wordShift(word_shift) + _wordShift(word_shift), + _docid_limit(docid_limit) { } @@ -32,4 +33,10 @@ SingleSmallNumericSearchContext::createFilterIterator(fef::TermFieldMatchData* m : std::make_unique<AttributeIteratorT<SingleSmallNumericSearchContext>>(*this, matchData); } +uint32_t +SingleSmallNumericSearchContext::get_committed_docid_limit() const noexcept +{ + return _docid_limit; +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.h index 46ed02b3eca..a42c8b9b29c 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.h @@ -22,6 +22,7 @@ private: uint32_t _valueShiftShift; uint32_t _valueShiftMask; uint32_t _wordShift; + uint32_t _docid_limit; int32_t onFind(DocId docId, int32_t elementId, int32_t & weight) const override { return find(docId, elementId, weight); @@ -32,7 +33,7 @@ private: } public: - SingleSmallNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const Word* word_data, Word value_mask, uint32_t value_shift_shift, uint32_t value_shift_mask, uint32_t word_shift); + SingleSmallNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const Word* word_data, Word value_mask, uint32_t value_shift_shift, uint32_t value_shift_mask, uint32_t word_shift, uint32_t docid_limit); int32_t find(DocId docId, int32_t elemId, int32_t & weight) const { if ( elemId != 0) return -1; @@ -53,6 +54,7 @@ public: std::unique_ptr<queryeval::SearchIterator> createFilterIterator(fef::TermFieldMatchData* matchData, bool strict) override; + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.cpp index 70023b27802..2d1748cefa5 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.cpp @@ -5,10 +5,10 @@ namespace search::attribute { -SingleStringEnumHintSearchContext::SingleStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<const char*>& enum_store, uint32_t doc_id_limit, uint64_t num_values) +SingleStringEnumHintSearchContext::SingleStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<const char*>& enum_store, uint64_t num_values) : SingleStringEnumSearchContext(std::move(qTerm), cased, toBeSearched, enum_indices, enum_store), EnumHintSearchContext(enum_store.get_dictionary(), - doc_id_limit, num_values) + enum_indices.size(), num_values) { setup_enum_hint_sc(enum_store, *this); } diff --git a/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.h index f9d44454cd0..f157bf17a71 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.h @@ -16,7 +16,7 @@ class SingleStringEnumHintSearchContext : public SingleStringEnumSearchContext, public EnumHintSearchContext { public: - SingleStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<const char*>& enum_store, uint32_t doc_id_limit, uint64_t num_values); + SingleStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<const char*>& enum_store, uint64_t num_values); ~SingleStringEnumHintSearchContext() override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.cpp index cba1d207501..8d23eaf7af0 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.cpp @@ -6,7 +6,7 @@ namespace search::attribute { -SingleStringEnumSearchContext::SingleStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<const char*>& enum_store) +SingleStringEnumSearchContext::SingleStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<const char*>& enum_store) : SingleEnumSearchContext<const char*, StringSearchContext>(StringMatcher(std::move(qTerm), cased), toBeSearched, enum_indices, enum_store) { } diff --git a/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.h index 6a9ed38b4ea..b8014b1b0e3 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.h @@ -14,7 +14,7 @@ namespace search::attribute { class SingleStringEnumSearchContext : public SingleEnumSearchContext<const char*, StringSearchContext> { public: - SingleStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<const char*>& enum_store); + SingleStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<const char*>& enum_store); SingleStringEnumSearchContext(SingleStringEnumSearchContext&&) noexcept; ~SingleStringEnumSearchContext() override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index 15fc819300c..87b7049b9b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -132,6 +132,7 @@ public: void fetchPostings(const queryeval::ExecuteInfo &execInfo) override; std::unique_ptr<queryeval::SearchIterator> createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) override; unsigned int approximateHits() const override; + uint32_t get_committed_docid_limit() const noexcept override; }; BitVectorSearchContext::BitVectorSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const SingleBoolAttribute & attr) @@ -177,6 +178,12 @@ BitVectorSearchContext::approximateHits() const { : 0; } +uint32_t +BitVectorSearchContext::get_committed_docid_limit() const noexcept +{ + return _doc_id_limit; +} + } std::unique_ptr<attribute::SearchContext> diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index c75ee0aacb5..606c7a92ef5 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -164,7 +164,7 @@ SingleValueNumericAttribute<B>::getSearch(QueryTermSimple::UP qTerm, { (void) params; QueryTermSimple::RangeResult<T> res = qTerm->getRange<T>(); - const T* data = &_data.acquire_elem_ref(0); + auto data = _data.make_read_view(this->getCommittedDocIdLimit()); if (res.isEqual()) { return std::make_unique<attribute::SingleNumericSearchContext<T, attribute::NumericMatcher<T>>>(std::move(qTerm), *this, data); } else { diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp index b840a0516b2..e459d3d9c9c 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp @@ -160,7 +160,8 @@ SingleValueNumericEnumAttribute<B>::getSearch(QueryTermSimple::UP qTerm, const attribute::SearchContextParams & params) const { (void) params; - return std::make_unique<attribute::SingleNumericEnumSearchContext<T>>(std::move(qTerm), *this, &this->_enumIndices.acquire_elem_ref(0), this->_enumStore); + auto docid_limit = this->getCommittedDocIdLimit(); + return std::make_unique<attribute::SingleNumericEnumSearchContext<T>>(std::move(qTerm), *this, this->_enumIndices.make_read_view(docid_limit), this->_enumStore); } } diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp index e353d03a9e8..a4b9abb084a 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp @@ -143,7 +143,8 @@ SingleValueNumericPostingAttribute<B>::getSearch(QueryTermSimple::UP qTerm, { using BaseSC = attribute::SingleNumericEnumSearchContext<T>; using SC = attribute::NumericPostingSearchContext<BaseSC, SelfType, vespalib::btree::BTreeNoLeafData>; - BaseSC base_sc(std::move(qTerm), *this, &this->_enumIndices.acquire_elem_ref(0), this->_enumStore); + auto docid_limit = this->getCommittedDocIdLimit(); + BaseSC base_sc(std::move(qTerm), *this, this->_enumIndices.make_read_view(docid_limit), this->_enumStore); return std::make_unique<SC>(std::move(base_sc), params, *this); } diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp index 13bf2f932e8..3c1621ac244 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp @@ -170,7 +170,8 @@ std::unique_ptr<attribute::SearchContext> SingleValueSmallNumericAttribute::getSearch(std::unique_ptr<QueryTermSimple> qTerm, const attribute::SearchContextParams &) const { - return std::make_unique<attribute::SingleSmallNumericSearchContext>(std::move(qTerm), *this, &_wordData.acquire_elem_ref(0), _valueMask, _valueShiftShift, _valueShiftMask, _wordShift); + auto docid_limit = getCommittedDocIdLimit(); + return std::make_unique<attribute::SingleSmallNumericSearchContext>(std::move(qTerm), *this, &_wordData.acquire_elem_ref(0), _valueMask, _valueShiftShift, _valueShiftMask, _wordShift, docid_limit); } void diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp index 69fe6435a03..c3f5c295260 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp @@ -46,7 +46,8 @@ SingleValueStringAttributeT<B>::getSearch(QueryTermSimpleUP qTerm, const attribute::SearchContextParams &) const { bool cased = this->get_match_is_cased(); - return std::make_unique<attribute::SingleStringEnumHintSearchContext>(std::move(qTerm), cased, *this, &this->_enumIndices.acquire_elem_ref(0), this->_enumStore, this->getCommittedDocIdLimit(), this->getStatus().getNumValues()); + auto docid_limit = this->getCommittedDocIdLimit(); + return std::make_unique<attribute::SingleStringEnumHintSearchContext>(std::move(qTerm), cased, *this, this->_enumIndices.make_read_view(docid_limit), this->_enumStore, this->getStatus().getNumValues()); } } diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp index 5b5214f6d3e..60847636baa 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp @@ -145,7 +145,8 @@ SingleValueStringPostingAttributeT<B>::getSearch(QueryTermSimpleUP qTerm, using BaseSC = attribute::SingleStringEnumSearchContext; using SC = attribute::StringPostingSearchContext<BaseSC, SelfType, vespalib::btree::BTreeNoLeafData>; bool cased = this->get_match_is_cased(); - BaseSC base_sc(std::move(qTerm), cased, *this, &this->_enumIndices.acquire_elem_ref(0), this->_enumStore); + auto docid_limit = this->getCommittedDocIdLimit(); + BaseSC base_sc(std::move(qTerm), cased, *this, this->_enumIndices.make_read_view(docid_limit), this->_enumStore); return std::make_unique<SC>(std::move(base_sc), params.useBitVector(), *this); diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt index a7c8d56f11d..089151455f3 100644 --- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt @@ -9,7 +9,6 @@ vespa_add_library(searchlib_common OBJECT condensedbitvectors.cpp documentlocations.cpp documentsummary.cpp - featureset.cpp fileheadercontext.cpp flush_token.cpp geo_gcd.cpp diff --git a/searchlib/src/vespa/searchlib/engine/searchreply.h b/searchlib/src/vespa/searchlib/engine/searchreply.h index 8f862d8dcf7..6b0edca3086 100644 --- a/searchlib/src/vespa/searchlib/engine/searchreply.h +++ b/searchlib/src/vespa/searchlib/engine/searchreply.h @@ -6,8 +6,8 @@ #include <vespa/document/base/globalid.h> #include <vespa/searchlib/common/hitrank.h> #include <vespa/searchlib/common/unique_issues.h> -#include <vespa/searchlib/common/featureset.h> #include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/util/featureset.h> #include <vector> namespace search::engine { @@ -15,6 +15,7 @@ namespace search::engine { class SearchReply { public: + using FeatureValues = vespalib::FeatureValues; using UP = std::unique_ptr<SearchReply>; class Hit diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h index 46b89fdfeb4..9bef389a278 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h @@ -109,7 +109,7 @@ public: uint32_t getArity() const { return _currArity; } uint32_t getNearDistance() const { return _extraIntArg1; } - uint32_t getTargetNumHits() const { return _extraIntArg1; } + uint32_t getTargetHits() const { return _extraIntArg1; } double getDistanceThreshold() const { return _extraDoubleArg4; } double getScoreThreshold() const { return _extraDoubleArg4; } double getThresholdBoostFactor() const { return _extraDoubleArg5; } diff --git a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp index d1c37cd6dcd..b2d8a0ee4be 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp @@ -1,15 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "nearest_neighbor_query_node.h" +#include <cassert> namespace search::streaming { -NearestNeighborQueryNode::NearestNeighborQueryNode(std::unique_ptr<QueryNodeResultBase> resultBase, const string& term, const string& index, int32_t id, search::query::Weight weight, double distance_threshold) - : QueryTerm(std::move(resultBase), term, index, Type::NEAREST_NEIGHBOR), +NearestNeighborQueryNode::NearestNeighborQueryNode(std::unique_ptr<QueryNodeResultBase> resultBase, + const string& query_tensor_name, const string& field_name, + uint32_t target_hits, double distance_threshold, + int32_t unique_id, search::query::Weight weight) + : QueryTerm(std::move(resultBase), query_tensor_name, field_name, Type::NEAREST_NEIGHBOR), + _target_hits(target_hits), _distance_threshold(distance_threshold), - _raw_score() + _distance(), + _calc() { - setUniqueId(id); + setUniqueId(unique_id); setWeight(weight); } @@ -18,13 +24,13 @@ NearestNeighborQueryNode::~NearestNeighborQueryNode() = default; bool NearestNeighborQueryNode::evaluate() const { - return _raw_score.has_value(); + return _distance.has_value(); } void NearestNeighborQueryNode::reset() { - _raw_score.reset(); + _distance.reset(); } NearestNeighborQueryNode* @@ -33,4 +39,14 @@ NearestNeighborQueryNode::as_nearest_neighbor_query_node() noexcept return this; } +std::optional<double> +NearestNeighborQueryNode::get_raw_score() const +{ + if (_distance.has_value()) { + assert(_calc != nullptr); + return _calc->to_raw_score(_distance.value()); + } + return std::nullopt; +} + } diff --git a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h index 0beb130c53d..c66364b0c52 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h +++ b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h @@ -8,16 +8,34 @@ namespace search::streaming { /* - * Nearest neighbor query node. + * Nearest neighbor query node for streaming search. */ class NearestNeighborQueryNode: public QueryTerm { +public: + class RawScoreCalculator { + public: + virtual ~RawScoreCalculator() = default; + /** + * Convert the given distance to a raw score. + * + * This is used during unpacking, and also signals that the entire document was a match. + */ + virtual double to_raw_score(double distance) = 0; + }; + private: + uint32_t _target_hits; double _distance_threshold; - // When this value is set it also indicates a match - std::optional<double> _raw_score; + // When this value is set it also indicates a match for this query node. + std::optional<double> _distance; + RawScoreCalculator* _calc; + public: - NearestNeighborQueryNode(std::unique_ptr<QueryNodeResultBase> resultBase, const string& term, const string& index, int32_t id, search::query::Weight weight, double distance_threshold); + NearestNeighborQueryNode(std::unique_ptr<QueryNodeResultBase> resultBase, + const string& query_tensor_name, const string& field_name, + uint32_t target_hits, double distance_threshold, + int32_t unique_id, search::query::Weight weight); NearestNeighborQueryNode(const NearestNeighborQueryNode &) = delete; NearestNeighborQueryNode & operator = (const NearestNeighborQueryNode &) = delete; NearestNeighborQueryNode(NearestNeighborQueryNode &&) = delete; @@ -27,9 +45,13 @@ public: void reset() override; NearestNeighborQueryNode* as_nearest_neighbor_query_node() noexcept override; const vespalib::string& get_query_tensor_name() const { return getTermString(); } + uint32_t get_target_hits() const { return _target_hits; } double get_distance_threshold() const { return _distance_threshold; } - void set_raw_score(double value) { _raw_score = value; } - const std::optional<double>& get_raw_score() const noexcept { return _raw_score; } + void set_raw_score_calc(RawScoreCalculator* calc_in) { _calc = calc_in; } + void set_distance(double value) { _distance = value; } + const std::optional<double>& get_distance() const { return _distance; } + // This is used during unpacking, and also signals to the RawScoreCalculator that the entire document was a match. + std::optional<double> get_raw_score() const; }; } diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp index 226cb92c894..84344831cbc 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp @@ -200,15 +200,17 @@ QueryNode::build_nearest_neighbor_query_node(const QueryNodeResultFactory& facto { vespalib::stringref query_tensor_name = query_rep.getTerm(); vespalib::stringref field_name = query_rep.getIndexName(); - int32_t id = query_rep.getUniqueId(); - search::query::Weight weight = query_rep.GetWeight(); + int32_t unique_id = query_rep.getUniqueId(); + auto weight = query_rep.GetWeight(); + uint32_t target_hits = query_rep.getTargetHits(); double distance_threshold = query_rep.getDistanceThreshold(); return std::make_unique<NearestNeighborQueryNode>(factory.create(), query_tensor_name, field_name, - id, - weight, - distance_threshold); + target_hits, + distance_threshold, + unique_id, + weight); } } diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h index 90bd87979c7..a552a650704 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h @@ -89,7 +89,7 @@ private: pureTermView = view; } else if (type == ParseItem::ITEM_WEAK_AND) { vespalib::stringref view = queryStack.getIndexName(); - uint32_t targetNumHits = queryStack.getTargetNumHits(); + uint32_t targetNumHits = queryStack.getTargetHits(); builder.addWeakAnd(arity, targetNumHits, view); pureTermView = view; } else if (type == ParseItem::ITEM_EQUIV) { @@ -134,7 +134,7 @@ private: vespalib::stringref view = queryStack.getIndexName(); int32_t id = queryStack.getUniqueId(); Weight weight = queryStack.GetWeight(); - uint32_t targetNumHits = queryStack.getTargetNumHits(); + uint32_t targetNumHits = queryStack.getTargetHits(); double scoreThreshold = queryStack.getScoreThreshold(); double thresholdBoostFactor = queryStack.getThresholdBoostFactor(); auto & wand = builder.addWandTerm(arity, view, id, weight, targetNumHits, scoreThreshold, thresholdBoostFactor); @@ -146,7 +146,7 @@ private: } else if (type == ParseItem::ITEM_NEAREST_NEIGHBOR) { vespalib::stringref query_tensor_name = queryStack.getTerm(); vespalib::stringref field_name = queryStack.getIndexName(); - uint32_t target_num_hits = queryStack.getTargetNumHits(); + uint32_t target_num_hits = queryStack.getTargetHits(); int32_t id = queryStack.getUniqueId(); Weight weight = queryStack.GetWeight(); bool allow_approximate = queryStack.getAllowApproximate(); diff --git a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp index c50c6ec49f5..86f520c8711 100644 --- a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp @@ -129,8 +129,16 @@ struct FakeContext : attribute::ISearchContext { DoubleRange getAsDoubleTerm() const override { abort(); } const QueryTermUCS4 * queryTerm() const override { abort(); } const vespalib::string &attributeName() const override { return name; } + uint32_t get_committed_docid_limit() const noexcept override; }; +uint32_t +FakeContext::get_committed_docid_limit() const noexcept +{ + auto& documents = result.inspect(); + return documents.empty() ? 0 : (documents.back().docId + 1); +} + } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 090042e5b83..2e874ffa4ae 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -30,10 +30,12 @@ vespa_add_library(searchlib_tensor OBJECT large_subspaces_buffer_type.cpp nearest_neighbor_index.cpp nearest_neighbor_index_saver.cpp + prenormalized_angular_distance.cpp serialized_fast_value_attribute.cpp serialized_tensor_ref.cpp small_subspaces_buffer_type.cpp subspace_type.cpp + temporary_vector_store.cpp tensor_attribute.cpp tensor_attribute_loader.cpp tensor_attribute_saver.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp index 5101373c047..efc1170baf5 100644 --- a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "angular_distance.h" +#include "temporary_vector_store.h" using vespalib::typify_invoke; using vespalib::eval::TypifyCellType; @@ -65,15 +66,15 @@ public: _tmpSpace(lhs.size), _lhs(_tmpSpace.storeLhs(lhs)) { - auto a = &_lhs[0]; + auto a = _lhs.data(); _lhs_norm_sq = _computer.dotProduct(a, a, lhs.size); } double calc(const vespalib::eval::TypedCells& rhs) const override { size_t sz = _lhs.size(); vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); assert(sz == rhs_vector.size()); - auto a = &_lhs[0]; - auto b = &rhs_vector[0]; + auto a = _lhs.data(); + auto b = rhs_vector.data(); double b_norm_sq = _computer.dotProduct(b, b, sz); double squared_norms = _lhs_norm_sq * b_norm_sq; double dot_product = _computer.dotProduct(a, b, sz); diff --git a/searchlib/src/vespa/searchlib/tensor/angular_distance.h b/searchlib/src/vespa/searchlib/tensor/angular_distance.h index 97f692d05a2..bba83576153 100644 --- a/searchlib/src/vespa/searchlib/tensor/angular_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/angular_distance.h @@ -60,8 +60,8 @@ public: auto rhs_vector = rhs.typify<FloatType>(); size_t sz = lhs_vector.size(); assert(sz == rhs_vector.size()); - auto a = &lhs_vector[0]; - auto b = &rhs_vector[0]; + auto a = lhs_vector.data(); + auto b = rhs_vector.data(); double a_norm_sq = _computer.dotProduct(a, a, sz); double b_norm_sq = _computer.dotProduct(b, b, sz); double squared_norms = a_norm_sq * b_norm_sq; diff --git a/searchlib/src/vespa/searchlib/tensor/bound_distance_function.cpp b/searchlib/src/vespa/searchlib/tensor/bound_distance_function.cpp index 56edbf9fede..33b94e5218c 100644 --- a/searchlib/src/vespa/searchlib/tensor/bound_distance_function.cpp +++ b/searchlib/src/vespa/searchlib/tensor/bound_distance_function.cpp @@ -1,58 +1,3 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bound_distance_function.h" -#include <vespa/log/log.h> - -LOG_SETUP(".searchlib.tensor.bound_distance_function"); - -using vespalib::ConstArrayRef; -using vespalib::ArrayRef; -using vespalib::eval::CellType; -using vespalib::eval::TypedCells; - -namespace search::tensor { - -namespace { - -template<typename FromType, typename ToType> -ConstArrayRef<ToType> -convert_cells(ArrayRef<ToType> space, TypedCells cells) -{ - assert(cells.size == space.size()); - auto old_cells = cells.typify<FromType>(); - ToType *p = space.data(); - for (FromType value : old_cells) { - ToType conv(value); - *p++ = conv; - } - return space; -} - -template <typename ToType> -struct ConvertCellsSelector -{ - template <typename FromType> static auto invoke(ArrayRef<ToType> dst, TypedCells src) { - return convert_cells<FromType, ToType>(dst, src); - } -}; - -} // namespace - -template <typename FloatType> -ConstArrayRef<FloatType> -TemporaryVectorStore<FloatType>::internal_convert(TypedCells cells, size_t offset) { - LOG_ASSERT(cells.size * 2 == _tmpSpace.size()); - ArrayRef<FloatType> where(_tmpSpace.data() + offset, cells.size); - using MyTypify = vespalib::eval::TypifyCellType; - using MySelector = ConvertCellsSelector<FloatType>; - ConstArrayRef<FloatType> result = vespalib::typify_invoke<1,MyTypify,MySelector>(cells.type, where, cells); - return result; -} - -template class TemporaryVectorStore<float>; -template class TemporaryVectorStore<double>; - -template class ConvertingBoundDistance<float>; -template class ConvertingBoundDistance<double>; - -} diff --git a/searchlib/src/vespa/searchlib/tensor/bound_distance_function.h b/searchlib/src/vespa/searchlib/tensor/bound_distance_function.h index 0f51e8a33ef..5d602a52227 100644 --- a/searchlib/src/vespa/searchlib/tensor/bound_distance_function.h +++ b/searchlib/src/vespa/searchlib/tensor/bound_distance_function.h @@ -5,7 +5,6 @@ #include <memory> #include <vespa/eval/eval/cell_type.h> #include <vespa/eval/eval/typed_cells.h> -#include <vespa/vespalib/util/array.h> #include <vespa/vespalib/util/arrayref.h> #include "distance_function.h" @@ -43,51 +42,4 @@ public: double limit) const = 0; }; - -/** helper class - temporary storage of possibly-converted vector cells */ -template <typename FloatType> -class TemporaryVectorStore { -private: - vespalib::Array<FloatType> _tmpSpace; - vespalib::ConstArrayRef<FloatType> internal_convert(vespalib::eval::TypedCells cells, size_t offset); -public: - TemporaryVectorStore(size_t vectorSize) : _tmpSpace(vectorSize * 2) {} - vespalib::ConstArrayRef<FloatType> storeLhs(vespalib::eval::TypedCells cells) { - return internal_convert(cells, 0); - } - vespalib::ConstArrayRef<FloatType> convertRhs(vespalib::eval::TypedCells cells) { - if (vespalib::eval::get_cell_type<FloatType>() == cells.type) [[likely]] { - return cells.unsafe_typify<FloatType>(); - } else { - return internal_convert(cells, cells.size); - } - } -}; - -template<typename FloatType> -class ConvertingBoundDistance : public BoundDistanceFunction { - mutable TemporaryVectorStore<FloatType> _tmpSpace; - const vespalib::eval::TypedCells _lhs; - const DistanceFunction &_df; -public: - ConvertingBoundDistance(const vespalib::eval::TypedCells& lhs, const DistanceFunction &df) - : BoundDistanceFunction(vespalib::eval::get_cell_type<FloatType>()), - _tmpSpace(lhs.size), - _lhs(_tmpSpace.storeLhs(lhs)), - _df(df) - {} - double calc(const vespalib::eval::TypedCells& rhs) const override { - return _df.calc(_lhs, vespalib::eval::TypedCells(_tmpSpace.convertRhs(rhs))); - } - double convert_threshold(double threshold) const override { - return _df.convert_threshold(threshold); - } - double to_rawscore(double distance) const override { - return _df.to_rawscore(distance); - } - double calc_with_limit(const vespalib::eval::TypedCells& rhs, double limit) const override { - return _df.calc_with_limit(_lhs, vespalib::eval::TypedCells(_tmpSpace.convertRhs(rhs)), limit); - } -}; - } diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp index 7ccca655943..4553f39a525 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp @@ -100,6 +100,27 @@ make_distance_function_factory(search::attribute::DistanceMetric variant, } return std::make_unique<AngularDistanceFunctionFactory<float>>(); } + if (variant == DistanceMetric::Euclidean) { + switch (cell_type) { + case CellType::DOUBLE: return std::make_unique<EuclideanDistanceFunctionFactory<double>>(); + case CellType::INT8: return std::make_unique<EuclideanDistanceFunctionFactory<vespalib::eval::Int8Float>>(); + default: return std::make_unique<EuclideanDistanceFunctionFactory<float>>(); + } + } + if (variant == DistanceMetric::PrenormalizedAngular) { + if (cell_type == CellType::DOUBLE) { + return std::make_unique<PrenormalizedAngularDistanceFunctionFactory<double>>(); + } + return std::make_unique<PrenormalizedAngularDistanceFunctionFactory<float>>(); + } + /* + if (variant == DistanceMetric::GeoDegrees) { + return std::make_unique<GeoDistanceFunctionFactory>(); + } + if (variant == DistanceMetric::Hamming) { + return std::make_unique<HammingDistanceFunctionFactory>(); + } + */ auto df = make_distance_function(variant, cell_type); return std::make_unique<SimpleDistanceFunctionFactory>(std::move(df)); } diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h index b28cc2bda46..2300dba2db1 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h @@ -8,3 +8,4 @@ #include "geo_degrees_distance.h" #include "hamming_distance.h" #include "inner_product_distance.h" +#include "prenormalized_angular_distance.h" diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp index c83f1821321..9c37b191637 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "euclidean_distance.h" +#include "temporary_vector_store.h" using vespalib::typify_invoke; using vespalib::eval::TypifyCellType; @@ -48,4 +49,73 @@ SquaredEuclideanDistance::calc_with_limit(const vespalib::eval::TypedCells& lhs, template class SquaredEuclideanDistanceHW<float>; template class SquaredEuclideanDistanceHW<double>; +using vespalib::eval::Int8Float; + +template<typename FloatType> +class BoundEuclideanDistance : public BoundDistanceFunction { +private: + const vespalib::hwaccelrated::IAccelrated & _computer; + mutable TemporaryVectorStore<FloatType> _tmpSpace; + const vespalib::ConstArrayRef<FloatType> _lhs_vector; + static const double *cast(const double * p) { return p; } + static const float *cast(const float * p) { return p; } + static const int8_t *cast(const Int8Float * p) { return reinterpret_cast<const int8_t *>(p); } +public: + BoundEuclideanDistance(const vespalib::eval::TypedCells& lhs) + : BoundDistanceFunction(vespalib::eval::get_cell_type<FloatType>()), + _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()), + _tmpSpace(lhs.size), + _lhs_vector(_tmpSpace.storeLhs(lhs)) + {} + double calc(const vespalib::eval::TypedCells& rhs) const override { + size_t sz = _lhs_vector.size(); + vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); + assert(sz == rhs_vector.size()); + auto a = _lhs_vector.data(); + auto b = rhs_vector.data(); + return _computer.squaredEuclideanDistance(cast(a), cast(b), sz); + } + double convert_threshold(double threshold) const override { + return threshold*threshold; + } + double to_rawscore(double distance) const override { + double d = sqrt(distance); + double score = 1.0 / (1.0 + d); + return score; + } + double calc_with_limit(const vespalib::eval::TypedCells& rhs, double limit) const override { + vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); + double sum = 0.0; + size_t sz = _lhs_vector.size(); + assert(sz == rhs_vector.size()); + for (size_t i = 0; i < sz && sum <= limit; ++i) { + double diff = _lhs_vector[i] - rhs_vector[i]; + sum += diff*diff; + } + return sum; + } +}; + +template class BoundEuclideanDistance<Int8Float>; +template class BoundEuclideanDistance<float>; +template class BoundEuclideanDistance<double>; + +template <typename FloatType> +BoundDistanceFunction::UP +EuclideanDistanceFunctionFactory<FloatType>::for_query_vector(const vespalib::eval::TypedCells& lhs) { + using DFT = BoundEuclideanDistance<FloatType>; + return std::make_unique<DFT>(lhs); +} + +template <typename FloatType> +BoundDistanceFunction::UP +EuclideanDistanceFunctionFactory<FloatType>::for_insertion_vector(const vespalib::eval::TypedCells& lhs) { + using DFT = BoundEuclideanDistance<FloatType>; + return std::make_unique<DFT>(lhs); +} + +template class EuclideanDistanceFunctionFactory<Int8Float>; +template class EuclideanDistanceFunctionFactory<float>; +template class EuclideanDistanceFunctionFactory<double>; + } diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h index 6505ea119ea..b406f0d3d1a 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h @@ -3,6 +3,7 @@ #pragma once #include "distance_function.h" +#include "distance_function_factory.h" #include <vespa/eval/eval/typed_cells.h> #include <vespa/vespalib/hwaccelrated/iaccelrated.h> #include <cmath> @@ -78,4 +79,15 @@ private: const vespalib::hwaccelrated::IAccelrated & _computer; }; + +template <typename FloatType> +class EuclideanDistanceFunctionFactory : public DistanceFunctionFactory { +public: + EuclideanDistanceFunctionFactory() + : DistanceFunctionFactory(vespalib::eval::get_cell_type<FloatType>()) + {} + BoundDistanceFunction::UP for_query_vector(const vespalib::eval::TypedCells& lhs) override; + BoundDistanceFunction::UP for_insertion_vector(const vespalib::eval::TypedCells& lhs) override; +}; + } diff --git a/searchlib/src/vespa/searchlib/tensor/inner_product_distance.h b/searchlib/src/vespa/searchlib/tensor/inner_product_distance.h index 8ba14580885..135bb186fd4 100644 --- a/searchlib/src/vespa/searchlib/tensor/inner_product_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/inner_product_distance.h @@ -54,7 +54,7 @@ public: auto rhs_vector = rhs.typify<FloatType>(); size_t sz = lhs_vector.size(); assert(sz == rhs_vector.size()); - double score = 1.0 - _computer.dotProduct(&lhs_vector[0], &rhs_vector[0], sz); + double score = 1.0 - _computer.dotProduct(lhs_vector.data(), rhs_vector.data(), sz); return std::max(0.0, score); } private: diff --git a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp new file mode 100644 index 00000000000..d2693f9f443 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp @@ -0,0 +1,82 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "prenormalized_angular_distance.h" +#include "temporary_vector_store.h" + +using vespalib::typify_invoke; +using vespalib::eval::TypifyCellType; + +namespace search::tensor { + +template<typename FloatType> +class BoundPrenormalizedAngularDistance : public BoundDistanceFunction { +private: + const vespalib::hwaccelrated::IAccelrated & _computer; + mutable TemporaryVectorStore<FloatType> _tmpSpace; + const vespalib::ConstArrayRef<FloatType> _lhs; + double _lhs_norm_sq; +public: + BoundPrenormalizedAngularDistance(const vespalib::eval::TypedCells& lhs) + : BoundDistanceFunction(vespalib::eval::get_cell_type<FloatType>()), + _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()), + _tmpSpace(lhs.size), + _lhs(_tmpSpace.storeLhs(lhs)) + { + auto a = _lhs.data(); + _lhs_norm_sq = _computer.dotProduct(a, a, lhs.size); + if (_lhs_norm_sq <= 0.0) { + _lhs_norm_sq = 1.0; + } + } + double calc(const vespalib::eval::TypedCells& rhs) const override { + size_t sz = _lhs.size(); + vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); + assert(sz == rhs_vector.size()); + auto a = _lhs.data(); + auto b = rhs_vector.data(); + double dot_product = _computer.dotProduct(a, b, sz); + double distance = _lhs_norm_sq - dot_product; + return distance; + } + double convert_threshold(double threshold) const override { + double cosine_similarity = 1.0 - threshold; + double dot_product = cosine_similarity * _lhs_norm_sq; + double distance = _lhs_norm_sq - dot_product; + return distance; + } + double to_rawscore(double distance) const override { + double dot_product = _lhs_norm_sq - distance; + double cosine_similarity = dot_product / _lhs_norm_sq; + // should be in in range [-1,1] but roundoff may cause problems: + cosine_similarity = std::min(1.0, cosine_similarity); + cosine_similarity = std::max(-1.0, cosine_similarity); + double cosine_distance = 1.0 - cosine_similarity; // in range [0,2] + double score = 1.0 / (1.0 + cosine_distance); + return score; + } + double calc_with_limit(const vespalib::eval::TypedCells& rhs, double) const override { + return calc(rhs); + } +}; + +template class BoundPrenormalizedAngularDistance<float>; +template class BoundPrenormalizedAngularDistance<double>; + +template <typename FloatType> +BoundDistanceFunction::UP +PrenormalizedAngularDistanceFunctionFactory<FloatType>::for_query_vector(const vespalib::eval::TypedCells& lhs) { + using DFT = BoundPrenormalizedAngularDistance<FloatType>; + return std::make_unique<DFT>(lhs); +} + +template <typename FloatType> +BoundDistanceFunction::UP +PrenormalizedAngularDistanceFunctionFactory<FloatType>::for_insertion_vector(const vespalib::eval::TypedCells& lhs) { + using DFT = BoundPrenormalizedAngularDistance<FloatType>; + return std::make_unique<DFT>(lhs); +} + +template class PrenormalizedAngularDistanceFunctionFactory<float>; +template class PrenormalizedAngularDistanceFunctionFactory<double>; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h new file mode 100644 index 00000000000..88953a236e7 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "distance_function.h" +#include "bound_distance_function.h" +#include "distance_function_factory.h" +#include <vespa/eval/eval/typed_cells.h> +#include <vespa/vespalib/hwaccelrated/iaccelrated.h> + +namespace search::tensor { + +/** + * Calculates inner-product "distance" between vectors with assumed norm 1. + * Should give same ordering as Angular distance, but is less expensive. + */ +template <typename FloatType> +class PrenormalizedAngularDistanceFunctionFactory : public DistanceFunctionFactory { +public: + PrenormalizedAngularDistanceFunctionFactory() + : DistanceFunctionFactory(vespalib::eval::get_cell_type<FloatType>()) + {} + BoundDistanceFunction::UP for_query_vector(const vespalib::eval::TypedCells& lhs) override; + BoundDistanceFunction::UP for_insertion_vector(const vespalib::eval::TypedCells& lhs) override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp new file mode 100644 index 00000000000..cc45f857d9f --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "temporary_vector_store.h" + +#include <vespa/log/log.h> + +LOG_SETUP(".searchlib.tensor.temporary_vector_store"); + +using vespalib::ConstArrayRef; +using vespalib::ArrayRef; +using vespalib::eval::CellType; +using vespalib::eval::TypedCells; + +namespace search::tensor { + +namespace { + +template<typename FromType, typename ToType> +ConstArrayRef<ToType> +convert_cells(ArrayRef<ToType> space, TypedCells cells) +{ + assert(cells.size == space.size()); + auto old_cells = cells.typify<FromType>(); + ToType *p = space.data(); + for (FromType value : old_cells) { + ToType conv(value); + *p++ = conv; + } + return space; +} + +template <typename ToType> +struct ConvertCellsSelector +{ + template <typename FromType> static auto invoke(ArrayRef<ToType> dst, TypedCells src) { + return convert_cells<FromType, ToType>(dst, src); + } +}; + +} // namespace + +template <typename FloatType> +ConstArrayRef<FloatType> +TemporaryVectorStore<FloatType>::internal_convert(TypedCells cells, size_t offset) { + LOG_ASSERT(cells.size * 2 == _tmpSpace.size()); + ArrayRef<FloatType> where(_tmpSpace.data() + offset, cells.size); + using MyTypify = vespalib::eval::TypifyCellType; + using MySelector = ConvertCellsSelector<FloatType>; + ConstArrayRef<FloatType> result = vespalib::typify_invoke<1,MyTypify,MySelector>(cells.type, where, cells); + return result; +} + +template class TemporaryVectorStore<vespalib::eval::Int8Float>; +template class TemporaryVectorStore<float>; +template class TemporaryVectorStore<double>; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.h b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.h new file mode 100644 index 00000000000..cd816621f91 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.h @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> +#include <vespa/eval/eval/cell_type.h> +#include <vespa/eval/eval/typed_cells.h> +#include <vespa/vespalib/util/arrayref.h> + +namespace search::tensor { + +/** helper class - temporary storage of possibly-converted vector cells */ +template <typename FloatType> +class TemporaryVectorStore { +private: + std::vector<FloatType> _tmpSpace; + vespalib::ConstArrayRef<FloatType> internal_convert(vespalib::eval::TypedCells cells, size_t offset); +public: + TemporaryVectorStore(size_t vectorSize) : _tmpSpace(vectorSize * 2) {} + vespalib::ConstArrayRef<FloatType> storeLhs(vespalib::eval::TypedCells cells) { + return internal_convert(cells, 0); + } + vespalib::ConstArrayRef<FloatType> convertRhs(vespalib::eval::TypedCells cells) { + if (vespalib::eval::get_cell_type<FloatType>() == cells.type) [[likely]] { + return cells.unsafe_typify<FloatType>(); + } else { + return internal_convert(cells, cells.size); + } + } +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index 2c644a243c8..a765208cb9e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -3,9 +3,9 @@ #pragma once #include "getdocsumargs.h" -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/common/geo_location_spec.h> #include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/util/featureset.h> #include <vespa/vespalib/util/stash.h> namespace juniper { @@ -48,6 +48,7 @@ protected: class GetDocsumsState { public: + using FeatureSet = vespalib::FeatureSet; const search::attribute::IAttributeVector * getAttribute(size_t index) const { return _attributes[index]; } GetDocsumArgs _args; // from getdocsums request diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index bad1ad5a6f3..c5e823bf9f4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -5,6 +5,8 @@ #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> +using vespalib::FeatureSet; + namespace search::docsummary { RankFeaturesDFW::RankFeaturesDFW() = default; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index a680b01d887..a1b2d6b3af6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -5,8 +5,7 @@ #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.docsummary.summaryfeaturesdfw"); +using vespalib::FeatureSet; namespace search::docsummary { diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt index c59b56eb68f..11bbf2c241a 100644 --- a/storage/src/tests/distributor/CMakeLists.txt +++ b/storage/src/tests/distributor/CMakeLists.txt @@ -19,7 +19,6 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST externaloperationhandlertest.cpp garbagecollectiontest.cpp getoperationtest.cpp - gtest_runner.cpp idealstatemanagertest.cpp joinbuckettest.cpp maintenancemocks.cpp @@ -27,6 +26,7 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST mergelimitertest.cpp mergeoperationtest.cpp multi_thread_stripe_access_guard_test.cpp + newest_replica_test.cpp node_supported_features_repo_test.cpp nodeinfotest.cpp nodemaintenancestatstrackertest.cpp @@ -57,7 +57,7 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST storage_testcommon storage_testhostreporter storage - GTest::GTest + GTest::gmock_main ) vespa_add_test( diff --git a/storage/src/tests/distributor/gtest_runner.cpp b/storage/src/tests/distributor/gtest_runner.cpp deleted file mode 100644 index 7c20f681093..00000000000 --- a/storage/src/tests/distributor/gtest_runner.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/gtest/gtest.h> - -#include <vespa/log/log.h> -LOG_SETUP("storage_distributor_gtest_runner"); - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/storage/src/tests/distributor/newest_replica_test.cpp b/storage/src/tests/distributor/newest_replica_test.cpp new file mode 100644 index 00000000000..9267c6e37a2 --- /dev/null +++ b/storage/src/tests/distributor/newest_replica_test.cpp @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/storage/distributor/operations/external/newest_replica.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/gtest/matchers/elements_are_distinct.h> + +using namespace ::testing; +using storage::api::Timestamp; +using document::BucketId; + +namespace storage::distributor { + +TEST(NewestReplicaTest, equality_predicate_considers_all_fields) { + std::vector elems = { + NewestReplica::of(Timestamp(1000), BucketId(16, 1), 0, false, false), + NewestReplica::of(Timestamp(1001), BucketId(16, 1), 0, false, false), + NewestReplica::of(Timestamp(1000), BucketId(16, 2), 0, false, false), + NewestReplica::of(Timestamp(1000), BucketId(16, 1), 1, false, false), + NewestReplica::of(Timestamp(1000), BucketId(16, 1), 0, true, false), + NewestReplica::of(Timestamp(1000), BucketId(16, 1), 0, false, true) + }; + EXPECT_THAT(elems, ElementsAreDistinct()); +} + +} diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp index dea215f47dc..aa5ad217ae9 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp @@ -15,13 +15,12 @@ #include <algorithm> #include <vespa/log/log.h> -LOG_SETUP(".distributor.callback.doc.put"); +LOG_SETUP(".distributor.operations.external.put"); - -using namespace storage::distributor; -using namespace storage; using document::BucketSpace; +namespace storage::distributor { + PutOperation::PutOperation(const DistributorNodeContext& node_ctx, DistributorStripeOperationContext& op_ctx, DistributorBucketSpace& bucketSpace, @@ -116,6 +115,20 @@ bool PutOperation::has_unavailable_targets_in_pending_state(const OperationTarge }); } +bool PutOperation::at_least_one_storage_node_is_available() const { + const lib::ClusterState& cluster_state = _bucketSpace.getClusterState(); + + const uint16_t storage_node_index_ubound = cluster_state.getNodeCount(lib::NodeType::STORAGE); + for (uint16_t i = 0; i < storage_node_index_ubound; i++) { + if (cluster_state.getNodeState(lib::Node(lib::NodeType::STORAGE, i)) + .getState().oneOf(storage_node_up_states())) + { + return true; + } + } + return false; +} + void PutOperation::onStart(DistributorStripeMessageSender& sender) { @@ -124,19 +137,7 @@ PutOperation::onStart(DistributorStripeMessageSender& sender) LOG(debug, "Received PUT %s for bucket %s", _msg->getDocumentId().toString().c_str(), bid.toString().c_str()); - lib::ClusterState systemState = _bucketSpace.getClusterState(); - - // Don't do anything if all nodes are down. - bool up = false; - for (uint16_t i = 0; i < systemState.getNodeCount(lib::NodeType::STORAGE); i++) { - if (systemState.getNodeState(lib::Node(lib::NodeType::STORAGE, i)) - .getState().oneOf(storage_node_up_states())) - { - up = true; - } - } - - if (up) { + if (at_least_one_storage_node_is_available()) { std::vector<document::BucketId> bucketsToCheckForSplit; OperationTargetResolverImpl targetResolver(_bucketSpace, _bucketSpace.getBucketDatabase(), @@ -145,8 +146,8 @@ PutOperation::onStart(DistributorStripeMessageSender& sender) _msg->getBucket().getBucketSpace()); OperationTargetList targets(targetResolver.getTargets(OperationTargetResolver::PUT, bid)); - for (size_t i = 0; i < targets.size(); ++i) { - if (_op_ctx.has_pending_message(targets[i].getNode().getIndex(), targets[i].getBucket(), + for (const auto& target : targets) { + if (_op_ctx.has_pending_message(target.getNode().getIndex(), target.getBucket(), api::MessageType::DELETEBUCKET_ID)) { _tracker.fail(sender, api::ReturnCode(api::ReturnCode::BUCKET_DELETED, @@ -179,13 +180,12 @@ PutOperation::onStart(DistributorStripeMessageSender& sender) std::vector<PersistenceMessageTracker::ToSend> putBatch; // Now send PUTs - for (uint32_t i = 0; i < targets.size(); i++) { - const OperationTarget& target(targets[i]); + for (const auto& target : targets) { sendPutToBucketOnNode(_msg->getBucket().getBucketSpace(), target.getBucketId(), target.getNode().getIndex(), putBatch); } - if (putBatch.size()) { + if (!putBatch.empty()) { _tracker.queueMessageBatch(putBatch); } else { const char* error = "Can't store document: No storage nodes available"; @@ -196,9 +196,9 @@ PutOperation::onStart(DistributorStripeMessageSender& sender) // Check whether buckets are large enough to be split. // TODO(vekterli): only check entries for sendToExisting? - for (uint32_t i = 0; i < entries.size(); ++i) { + for (const auto& entry : entries) { _op_ctx.send_inline_split_if_bucket_too_large(_msg->getBucket().getBucketSpace(), - entries[i], _msg->getPriority()); + entry, _msg->getPriority()); } _tracker.flushQueue(sender); @@ -235,3 +235,5 @@ PutOperation::onClose(DistributorStripeMessageSender& sender) LOG(debug, "%s", error); _tracker.fail(sender, api::ReturnCode(api::ReturnCode::ABORTED, error)); } + +} diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.h b/storage/src/vespa/storage/distributor/operations/external/putoperation.h index 9801fed0c99..283395f1406 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.h @@ -47,9 +47,10 @@ private: void sendPutToBucketOnNode(document::BucketSpace bucketSpace, const document::BucketId& bucketId, const uint16_t node, std::vector<PersistenceMessageTracker::ToSend>& putBatch); - bool shouldImplicitlyActivateReplica(const OperationTargetList& targets) const; + [[nodiscard]] bool shouldImplicitlyActivateReplica(const OperationTargetList& targets) const; - bool has_unavailable_targets_in_pending_state(const OperationTargetList& targets) const; + [[nodiscard]] bool has_unavailable_targets_in_pending_state(const OperationTargetList& targets) const; + [[nodiscard]] bool at_least_one_storage_node_is_available() const; std::shared_ptr<api::PutCommand> _msg; DistributorStripeOperationContext& _op_ctx; diff --git a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp index 584ad9de2ce..b752a7fadde 100644 --- a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp @@ -5,13 +5,12 @@ #include <vespa/vdslib/state/clusterstate.h> #include <vespa/log/log.h> -LOG_SETUP(".distributor.operation.external.remove"); +LOG_SETUP(".distributor.operations.external.remove"); - -using namespace storage::distributor; -using namespace storage; using document::BucketSpace; +namespace storage::distributor { + RemoveOperation::RemoveOperation(const DistributorNodeContext& node_ctx, DistributorStripeOperationContext& op_ctx, DistributorBucketSpace& bucketSpace, @@ -100,3 +99,5 @@ RemoveOperation::onClose(DistributorStripeMessageSender& sender) { _tracker.fail(sender, api::ReturnCode(api::ReturnCode::ABORTED, "Process is shutting down")); } + +} diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index 9ae6aaf0653..e7eb7a752fb 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -8,11 +8,9 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/storage/common/bucket_resolver.h> #include <vespa/storageapi/message/datagram.h> -#include <vespa/storageapi/message/documentsummary.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/queryresult.h> #include <vespa/storageapi/message/removelocation.h> -#include <vespa/storageapi/message/searchresult.h> #include <vespa/storageapi/message/stat.h> #include <vespa/storageapi/message/visitor.h> #include <vespa/messagebus/error.h> @@ -237,24 +235,12 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg) toMsg = std::move(to); break; } - case api::MessageType::SEARCHRESULT_ID: - { - auto & from(static_cast<api::SearchResultCommand&>(fromMsg)); - toMsg = std::make_unique<documentapi::SearchResultMessage>(std::move(from)); - break; - } case api::MessageType::QUERYRESULT_ID: { auto & from(static_cast<api::QueryResultCommand&>(fromMsg)); toMsg = std::make_unique<documentapi::QueryResultMessage>(std::move(from.getSearchResult()), from.getDocumentSummary()); break; } - case api::MessageType::DOCUMENTSUMMARY_ID: - { - auto & from(static_cast<api::DocumentSummaryCommand&>(fromMsg)); - toMsg = std::make_unique<documentapi::DocumentSummaryMessage>(from); - break; - } case api::MessageType::MAPVISITOR_ID: { auto & from(static_cast<api::MapVisitorCommand&>(fromMsg)); diff --git a/storage/src/vespa/storageapi/message/CMakeLists.txt b/storage/src/vespa/storageapi/message/CMakeLists.txt index 2a761921dff..2728b5b51ad 100644 --- a/storage/src/vespa/storageapi/message/CMakeLists.txt +++ b/storage/src/vespa/storageapi/message/CMakeLists.txt @@ -6,9 +6,7 @@ vespa_add_library(storageapi_message OBJECT bucket.cpp visitor.cpp state.cpp - searchresult.cpp bucketsplitting.cpp - documentsummary.cpp stat.cpp removelocation.cpp queryresult.cpp diff --git a/storage/src/vespa/storageapi/message/documentsummary.cpp b/storage/src/vespa/storageapi/message/documentsummary.cpp deleted file mode 100644 index 6909b4d223c..00000000000 --- a/storage/src/vespa/storageapi/message/documentsummary.cpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "documentsummary.h" -#include <ostream> - -namespace storage { -namespace api { - -IMPLEMENT_COMMAND(DocumentSummaryCommand, DocumentSummaryReply) -IMPLEMENT_REPLY(DocumentSummaryReply) - -DocumentSummaryCommand::DocumentSummaryCommand() - : StorageCommand(MessageType::DOCUMENTSUMMARY), - DocumentSummary() -{ } - -void -DocumentSummaryCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "DocumentSummary(" << getSummaryCount() << " summaries)"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -DocumentSummaryReply::DocumentSummaryReply(const DocumentSummaryCommand& cmd) - : StorageReply(cmd) -{ } - -void -DocumentSummaryReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "DocumentSummaryReply()"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -} // api -} // storage diff --git a/storage/src/vespa/storageapi/message/documentsummary.h b/storage/src/vespa/storageapi/message/documentsummary.h deleted file mode 100644 index 5e2c1af3cfd..00000000000 --- a/storage/src/vespa/storageapi/message/documentsummary.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include <vespa/vdslib/container/documentsummary.h> - -namespace storage { -namespace api { - -/** - * @class DocumentSummaryCommand - * @ingroup message - * - * @brief The result of a searchvisitor. - */ -class DocumentSummaryCommand : public StorageCommand, - public vdslib::DocumentSummary -{ -public: - explicit DocumentSummaryCommand(); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(DocumentSummaryCommand, onDocumentSummary) -}; - -/** - * @class DocumentSummaryReply - * @ingroup message - * - * @brief Response to a document summary command. - */ -class DocumentSummaryReply : public StorageReply { -public: - explicit DocumentSummaryReply(const DocumentSummaryCommand& command); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(DocumentSummaryReply, onDocumentSummaryReply) -}; - -} // api -} // storage diff --git a/storage/src/vespa/storageapi/message/searchresult.cpp b/storage/src/vespa/storageapi/message/searchresult.cpp deleted file mode 100644 index b2cf04b0410..00000000000 --- a/storage/src/vespa/storageapi/message/searchresult.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "searchresult.h" -#include <ostream> - -using vdslib::SearchResult; - -namespace storage { -namespace api { - -IMPLEMENT_COMMAND(SearchResultCommand, SearchResultReply) -IMPLEMENT_REPLY(SearchResultReply) - -SearchResultCommand::SearchResultCommand() - : StorageCommand(MessageType::SEARCHRESULT), - SearchResult() -{ -} - -void -SearchResultCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SearchResultCommand(" << getHitCount() << " hits)"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -SearchResultReply::SearchResultReply(const SearchResultCommand& cmd) - : StorageReply(cmd) -{ } - -void -SearchResultReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SearchResultReply()"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -} // api -} // storage diff --git a/storage/src/vespa/storageapi/message/searchresult.h b/storage/src/vespa/storageapi/message/searchresult.h deleted file mode 100644 index b12fa5e1613..00000000000 --- a/storage/src/vespa/storageapi/message/searchresult.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include <vespa/vdslib/container/searchresult.h> - -namespace storage::api { - -/** - * @class SearchResultCommand - * @ingroup message - * - * @brief The result of a searchvisitor. - */ -class SearchResultCommand : public StorageCommand, public vdslib::SearchResult { -public: - SearchResultCommand(); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(SearchResultCommand, onSearchResult) -}; - -/** - * @class SearchResultReply - * @ingroup message - * - * @brief Response to a search result command. - */ -class SearchResultReply : public StorageReply { -public: - explicit SearchResultReply(const SearchResultCommand& command); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(SearchResultReply, onSearchResultReply) -}; - -} diff --git a/storage/src/vespa/storageapi/messageapi/messagehandler.h b/storage/src/vespa/storageapi/messageapi/messagehandler.h index 9ba8542e9db..fa362d5380f 100644 --- a/storage/src/vespa/storageapi/messageapi/messagehandler.h +++ b/storage/src/vespa/storageapi/messageapi/messagehandler.h @@ -29,8 +29,6 @@ class CreateVisitorCommand; // Create a new visitor class DestroyVisitorCommand; // Destroy a running visitor class VisitorInfoCommand; // Sends visitor info to visitor controller class MapVisitorCommand; -class SearchResultCommand; -class DocumentSummaryCommand; class QueryResultCommand; class InternalCommand; @@ -67,8 +65,6 @@ class CreateVisitorReply; class DestroyVisitorReply; class VisitorInfoReply; class MapVisitorReply; -class SearchResultReply; -class DocumentSummaryReply; class QueryResultReply; class InternalReply; @@ -137,12 +133,8 @@ public: virtual bool onVisitorInfoReply(const std::shared_ptr<api::VisitorInfoReply>&) { return false; } virtual bool onMapVisitor(const std::shared_ptr<api::MapVisitorCommand>&) { return false; } virtual bool onMapVisitorReply(const std::shared_ptr<api::MapVisitorReply>&) { return false; } - virtual bool onSearchResult(const std::shared_ptr<api::SearchResultCommand>&) { return false; } - virtual bool onSearchResultReply(const std::shared_ptr<api::SearchResultReply>&) { return false; } virtual bool onQueryResult(const std::shared_ptr<api::QueryResultCommand>&) { return false; } virtual bool onQueryResultReply(const std::shared_ptr<api::QueryResultReply>&) { return false; } - virtual bool onDocumentSummary(const std::shared_ptr<api::DocumentSummaryCommand>&) { return false; } - virtual bool onDocumentSummaryReply(const std::shared_ptr<api::DocumentSummaryReply>&) { return false; } virtual bool onEmptyBuckets(const std::shared_ptr<api::EmptyBucketsCommand>&) { return false; } virtual bool onEmptyBucketsReply(const std::shared_ptr<api::EmptyBucketsReply>&) { return false; } virtual bool onInternal(const std::shared_ptr<api::InternalCommand>&) { return false; } diff --git a/storage/src/vespa/storageapi/messageapi/storagemessage.cpp b/storage/src/vespa/storageapi/messageapi/storagemessage.cpp index b1d68fd77e3..c72ece80476 100644 --- a/storage/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storage/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -76,10 +76,6 @@ const MessageType MessageType::APPLYBUCKETDIFF("ApplyBucketDiff", APPLYBUCKETDIF const MessageType MessageType::APPLYBUCKETDIFF_REPLY("ApplyBucketDiff reply", APPLYBUCKETDIFF_REPLY_ID, &MessageType::APPLYBUCKETDIFF); const MessageType MessageType::VISITOR_INFO("VisitorInfo", VISITOR_INFO_ID); const MessageType MessageType::VISITOR_INFO_REPLY("VisitorInfo reply", VISITOR_INFO_REPLY_ID, &MessageType::VISITOR_INFO); -const MessageType MessageType::SEARCHRESULT("SearchResult", SEARCHRESULT_ID); -const MessageType MessageType::SEARCHRESULT_REPLY("SearchResult reply", SEARCHRESULT_REPLY_ID, &MessageType::SEARCHRESULT); -const MessageType MessageType::DOCUMENTSUMMARY("DocumentSummary", DOCUMENTSUMMARY_ID); -const MessageType MessageType::DOCUMENTSUMMARY_REPLY("DocumentSummary reply", DOCUMENTSUMMARY_REPLY_ID, &MessageType::DOCUMENTSUMMARY); const MessageType MessageType::MAPVISITOR("Mapvisitor", MAPVISITOR_ID); const MessageType MessageType::MAPVISITOR_REPLY("Mapvisitor reply", MAPVISITOR_REPLY_ID, &MessageType::MAPVISITOR); const MessageType MessageType::SPLITBUCKET("SplitBucket", SPLITBUCKET_ID); diff --git a/storage/src/vespa/storageapi/messageapi/storagemessage.h b/storage/src/vespa/storageapi/messageapi/storagemessage.h index 282f110646d..4649781c1e5 100644 --- a/storage/src/vespa/storageapi/messageapi/storagemessage.h +++ b/storage/src/vespa/storageapi/messageapi/storagemessage.h @@ -122,14 +122,14 @@ public: DOCBLOCK_REPLY_ID = 59, VISITOR_INFO_ID = 60, VISITOR_INFO_REPLY_ID = 61, - SEARCHRESULT_ID = 64, - SEARCHRESULT_REPLY_ID = 65, + // SEARCHRESULT_ID = 64, + // SEARCHRESULT_REPLY_ID = 65, SPLITBUCKET_ID = 66, SPLITBUCKET_REPLY_ID = 67, JOINBUCKETS_ID = 68, JOINBUCKETS_REPLY_ID = 69, - DOCUMENTSUMMARY_ID = 72, - DOCUMENTSUMMARY_REPLY_ID = 73, + // DOCUMENTSUMMARY_ID = 72, + // DOCUMENTSUMMARY_REPLY_ID = 73, MAPVISITOR_ID = 74, MAPVISITOR_REPLY_ID = 75, STATBUCKET_ID = 76, @@ -208,14 +208,10 @@ public: static const MessageType APPLYBUCKETDIFF_REPLY; static const MessageType VISITOR_INFO; static const MessageType VISITOR_INFO_REPLY; - static const MessageType SEARCHRESULT; - static const MessageType SEARCHRESULT_REPLY; static const MessageType SPLITBUCKET; static const MessageType SPLITBUCKET_REPLY; static const MessageType JOINBUCKETS; static const MessageType JOINBUCKETS_REPLY; - static const MessageType DOCUMENTSUMMARY; - static const MessageType DOCUMENTSUMMARY_REPLY; static const MessageType MAPVISITOR; static const MessageType MAPVISITOR_REPLY; static const MessageType STATBUCKET; diff --git a/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp b/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp index 6950c90f097..791ec01162f 100644 --- a/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp +++ b/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp @@ -285,7 +285,7 @@ HitCollectorTest::testFeatureSet() FeatureResolver resolver(rankProgram.get_resolver()); search::StringStringMap renames; renames["bar"] = "qux"; - search::FeatureSet::SP sf = hc.getFeatureSet(rankProgram, resolver, renames); + vespalib::FeatureSet::SP sf = hc.getFeatureSet(rankProgram, resolver, renames); EXPECT_EQUAL(sf->getNames().size(), 3u); EXPECT_EQUAL(sf->getNames()[0], "foo"); diff --git a/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp b/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp index 43c77398be8..b64d477fd4c 100644 --- a/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp +++ b/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp @@ -31,9 +31,11 @@ struct MockQuery { std::vector<std::unique_ptr<NearestNeighborQueryNode>> nodes; QueryTermList term_list; MockQuery& add(const vespalib::string& query_tensor_name, + uint32_t target_hits, double distance_threshold) { std::unique_ptr<QueryNodeResultBase> base; - auto node = std::make_unique<NearestNeighborQueryNode>(std::move(base), query_tensor_name, "my_tensor_field", 7, search::query::Weight(11), distance_threshold); + auto node = std::make_unique<NearestNeighborQueryNode>(std::move(base), query_tensor_name, "my_tensor_field", + target_hits, distance_threshold, 7, search::query::Weight(100)); nodes.push_back(std::move(node)); term_list.push_back(nodes.back().get()); return *this; @@ -90,34 +92,71 @@ public: query.reset(); searcher.onValue(fv); } + void expect_match(const vespalib::string& spec_expr, double exp_square_distance, const NearestNeighborQueryNode& node) { + match(spec_expr); + expect_match(exp_square_distance, node); + } void expect_match(double exp_square_distance, const NearestNeighborQueryNode& node) { double exp_raw_score = dist_func.to_rawscore(exp_square_distance); EXPECT_TRUE(node.evaluate()); + EXPECT_DOUBLE_EQ(exp_square_distance, node.get_distance().value()); EXPECT_DOUBLE_EQ(exp_raw_score, node.get_raw_score().value()); } + void expect_not_match(const vespalib::string& spec_expr, const NearestNeighborQueryNode& node) { + match(spec_expr); + EXPECT_FALSE(node.evaluate()); + } }; -TEST_F(NearestNeighborSearcherTest, raw_score_calculated_with_distance_threshold) +TEST_F(NearestNeighborSearcherTest, distance_heap_keeps_the_best_target_hits) { - query.add("qt1", 3); + query.add("qt1", 2, 100.0); + const auto& node = query.get(0); set_query_tensor("qt1", "tensor(x[2]):[1,3]"); prepare(); - match("tensor(x[2]):[1,5]"); - expect_match((5-3)*(5-3), query.get(0)); + expect_match("tensor(x[2]):[1,7]", (7-3)*(7-3), node); + expect_match("tensor(x[2]):[1,9]", (9-3)*(9-3), node); - match("tensor(x[2]):[1,6]"); - expect_match((6-3)*(6-3), query.get(0)); + // The distance limit is now (9-3)*(9-3) = 36, so this is not good enough. + expect_not_match("tensor(x[2]):[1,10]", node); + + expect_match("tensor(x[2]):[1,5]", (5-3)*(5-3), node); + + // The distance limit is now (7-3)*(7-3) = 16, so this is not good enough. + expect_not_match("tensor(x[2]):[1,8]", node); + + // This is not considered a document match as get_raw_score() is not called, + // and the distance heap is not updated. + match("tensor(x[2]):[1,4]"); + EXPECT_EQ(1, node.get_distance().value()); + EXPECT_TRUE(node.evaluate()); + + // The distance limit is still (7-3)*(7-3) = 16, so this is in fact good enough. + expect_match("tensor(x[2]):[1,6]", (6-3)*(6-3), node); + + // The distance limit is (6-3)*(6-3) = 4, and a similar distance is a match. + expect_match("tensor(x[2]):[1,6]", (6-3)*(6-3), node); +} + +TEST_F(NearestNeighborSearcherTest, raw_score_calculated_with_distance_threshold) +{ + query.add("qt1", 10, 3.0); + const auto& node = query.get(0); + set_query_tensor("qt1", "tensor(x[2]):[1,3]"); + prepare(); + + expect_match("tensor(x[2]):[1,5]", (5-3)*(5-3), node); + expect_match("tensor(x[2]):[1,6]", (6-3)*(6-3), node); - match("tensor(x[2]):[1,7]"); // This is not a match since ((7-3)*(7-3) = 16) is larger than the internal distance threshold of (3*3 = 9). - EXPECT_FALSE(query.get(0).evaluate()); + expect_not_match("tensor(x[2]):[1,7]", node); } TEST_F(NearestNeighborSearcherTest, raw_score_calculated_for_two_query_operators) { - query.add("qt1", 3); - query.add("qt2", 4); + query.add("qt1", 10, 3.0); + query.add("qt2", 10, 4.0); set_query_tensor("qt1", "tensor(x[2]):[1,3]"); set_query_tensor("qt2", "tensor(x[2]):[1,4]"); prepare(); diff --git a/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp b/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp index 9f3f3d770e4..4d425d9dedd 100644 --- a/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp +++ b/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp @@ -55,6 +55,10 @@ RankProcessorTest::build_query(QueryBuilder<SimpleQueryNodeTypes> &builder) _query_wrapper = std::make_unique<QueryWrapper>(*_query); } +class MockRawScoreCalculator : public search::streaming::NearestNeighborQueryNode::RawScoreCalculator { +public: + double to_raw_score(double distance) override { return distance * 2; } +}; TEST_F(RankProcessorTest, unpack_match_data_for_nearest_neighbor_query_node) { @@ -71,6 +75,8 @@ TEST_F(RankProcessorTest, unpack_match_data_for_nearest_neighbor_query_node) EXPECT_EQ(1u, term_list.size()); auto node = dynamic_cast<NearestNeighborQueryNode*>(term_list.front().getTerm()); EXPECT_NE(nullptr, node); + MockRawScoreCalculator calc; + node->set_raw_score_calc(&calc); auto& qtd = static_cast<QueryTermData &>(node->getQueryItem()); auto& td = qtd.getTermData(); constexpr TermFieldHandle handle = 27; @@ -82,11 +88,11 @@ TEST_F(RankProcessorTest, unpack_match_data_for_nearest_neighbor_query_node) EXPECT_EQ(invalid_id, tfmd->getDocId()); RankProcessor::unpack_match_data(1, *md, *_query_wrapper); EXPECT_EQ(invalid_id, tfmd->getDocId()); - constexpr double raw_score = 1.5; - node->set_raw_score(raw_score); + constexpr double distance = 1.5; + node->set_distance(distance); RankProcessor::unpack_match_data(2, *md, *_query_wrapper); EXPECT_EQ(2, tfmd->getDocId()); - EXPECT_EQ(raw_score, tfmd->getRawScore()); + EXPECT_EQ(distance * 2, tfmd->getRawScore()); node->reset(); RankProcessor::unpack_match_data(3, *md, *_query_wrapper); EXPECT_EQ(2, tfmd->getDocId()); diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp index 10e6c6aa68a..7b4e3cb0208 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp @@ -10,8 +10,8 @@ #include <vespa/log/log.h> LOG_SETUP(".searchvisitor.hitcollector"); -using search::FeatureSet; using search::fef::MatchData; +using vespalib::FeatureSet; using vdslib::SearchResult; namespace streaming { diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h index 6ce7459adfd..2918f815811 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h @@ -2,13 +2,13 @@ #pragma once -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/common/stringmap.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/vdslib/container/searchresult.h> #include <vespa/vsm/common/docsum.h> #include <vespa/vsm/common/storagedocument.h> #include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/featureset.h> namespace search { namespace fef { class FeatureResolver; } } @@ -132,9 +132,9 @@ public: * @param rankProgram the rank program used to calculate all features. * @param resolver feature resolver, gives feature names and values **/ - search::FeatureSet::SP getFeatureSet(IRankProgram &rankProgram, - const search::fef::FeatureResolver &resolver, - const search::StringStringMap &feature_rename_map); + vespalib::FeatureSet::SP getFeatureSet(IRankProgram &rankProgram, + const search::fef::FeatureResolver &resolver, + const search::StringStringMap &feature_rename_map); }; diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp index 538f3efe44a..81df2b5492f 100644 --- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp @@ -22,13 +22,16 @@ IndexEnvironment::IndexEnvironment(IndexEnvironment &&) noexcept = default; IndexEnvironment::~IndexEnvironment() = default; bool -IndexEnvironment::addField(const vespalib::string & name, bool isAttribute) +IndexEnvironment::addField(const vespalib::string& name, + bool isAttribute, + search::fef::FieldInfo::DataType data_type) { if (getFieldByName(name) != nullptr) { return false; } FieldInfo info(isAttribute ? FieldType::ATTRIBUTE : FieldType::INDEX, FieldInfo::CollectionType::SINGLE, name, _fields.size()); + info.set_data_type(data_type); info.addAttribute(); // we are able to produce needed attributes at query time _fields.push_back(info); _fieldNames[info.name()] = info.id(); diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h index af037d87076..ef679cacdf0 100644 --- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h @@ -83,7 +83,9 @@ public: return nullptr; } - bool addField(const vespalib::string & name, bool isAttribute); + bool addField(const vespalib::string& name, + bool isAttribute, + search::fef::FieldInfo::DataType data_type); search::fef::Properties & getProperties() { return _properties; } diff --git a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp index 706325a0f7a..81a2a48fb4d 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp @@ -2,6 +2,7 @@ #include "rankmanager.h" #include <vespa/searchlib/features/setup.h> +#include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/functiontablefactory.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exception.h> @@ -40,6 +41,16 @@ RankManager::Snapshot::addProperties(const vespa::config::search::RankProfilesCo } } +FieldInfo::DataType +to_data_type(VsmfieldsConfig::Fieldspec::Searchmethod search_method) +{ + if (search_method == VsmfieldsConfig::Fieldspec::Searchmethod::NEAREST_NEIGHBOR) { + return FieldInfo::DataType::TENSOR; + } + // This is the default FieldInfo data type if not specified. + return FieldInfo::DataType::DOUBLE; +} + void RankManager::Snapshot::detectFields(const VsmfieldsHandle & fields) { @@ -49,7 +60,7 @@ RankManager::Snapshot::detectFields(const VsmfieldsHandle & fields) LOG(debug, "Adding field of type '%s' and name '%s' with id '%u' the index environment.", isAttribute ? "ATTRIBUTE" : "INDEX", fs.name.c_str(), i); // This id must match the vsm specific field id - _protoEnv.addField(fs.name, isAttribute); + _protoEnv.addField(fs.name, isAttribute, to_data_type(fs.searchmethod)); } } diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index b41eb041c57..3ce137bffe5 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -10,7 +10,7 @@ #include <vespa/log/log.h> LOG_SETUP(".searchvisitor.rankprocessor"); -using search::FeatureSet; +using vespalib::FeatureSet; using search::fef::FeatureHandle; using search::fef::ITermData; using search::fef::ITermFieldData; @@ -56,24 +56,28 @@ RankProcessor::initQueryEnvironment() { QueryWrapper::TermList & terms = _query.getTermList(); - for (uint32_t i = 0; i < terms.size(); ++i) { - if (terms[i].isGeoPosTerm()) { - const vespalib::string & fieldName = terms[i].getTerm()->index(); - const vespalib::string & locStr = terms[i].getTerm()->getTermString(); + for (auto& term : terms) { + if (term.isGeoPosTerm()) { + const vespalib::string & fieldName = term.getTerm()->index(); + const vespalib::string & locStr = term.getTerm()->getTermString(); _queryEnv.addGeoLocation(fieldName, locStr); } - if (!terms[i].isPhraseTerm() || terms[i].isFirstPhraseTerm()) { // register 1 term data per phrase - QueryTermData & qtd = dynamic_cast<QueryTermData &>(terms[i].getTerm()->getQueryItem()); + if (!term.isPhraseTerm() || term.isFirstPhraseTerm()) { // register 1 term data per phrase + QueryTermData & qtd = dynamic_cast<QueryTermData &>(term.getTerm()->getQueryItem()); - qtd.getTermData().setWeight(terms[i].getTerm()->weight()); - qtd.getTermData().setUniqueId(terms[i].getTerm()->uniqueId()); - if (terms[i].isFirstPhraseTerm()) { - qtd.getTermData().setPhraseLength(terms[i].getParent()->width()); + qtd.getTermData().setWeight(term.getTerm()->weight()); + qtd.getTermData().setUniqueId(term.getTerm()->uniqueId()); + if (term.isFirstPhraseTerm()) { + qtd.getTermData().setPhraseLength(term.getParent()->width()); } else { qtd.getTermData().setPhraseLength(1); } + auto* nn_term = term.getTerm()->as_nearest_neighbor_query_node(); + if (nn_term != nullptr) { + qtd.getTermData().set_query_tensor_name(nn_term->get_query_tensor_name()); + } - vespalib::string expandedIndexName = vsm::FieldSearchSpecMap::stripNonFields(terms[i].getTerm()->index()); + vespalib::string expandedIndexName = vsm::FieldSearchSpecMap::stripNonFields(term.getTerm()->index()); const RankManager::View *view = _rankManagerSnapshot->getView(expandedIndexName); if (view != nullptr) { RankManager::View::const_iterator iter = view->begin(); @@ -83,17 +87,17 @@ RankProcessor::initQueryEnvironment() } } else { LOG(warning, "Could not find a view for index '%s'. Ranking no fields.", - getIndexName(terms[i].getTerm()->index(), expandedIndexName).c_str()); + getIndexName(term.getTerm()->index(), expandedIndexName).c_str()); } LOG(debug, "Setup query term '%s:%s' (%s)", - getIndexName(terms[i].getTerm()->index(), expandedIndexName).c_str(), - terms[i].getTerm()->getTerm(), - terms[i].isFirstPhraseTerm() ? "phrase" : "term"); + getIndexName(term.getTerm()->index(), expandedIndexName).c_str(), + term.getTerm()->getTerm(), + term.isFirstPhraseTerm() ? "phrase" : "term"); _queryEnv.addTerm(&qtd.getTermData()); } else { LOG(debug, "Ignore query term '%s:%s' (part of phrase)", - terms[i].getTerm()->index().c_str(), terms[i].getTerm()->getTerm()); + term.getTerm()->index().c_str(), term.getTerm()->getTerm()); } } _rankSetup.prepareSharedState(_queryEnv, _queryEnv.getObjectStore()); @@ -237,7 +241,7 @@ RankProcessor::unpack_match_data(uint32_t docid, MatchData &matchData, QueryWrap for (QueryWrapper::Term & term: query.getTermList()) { auto nn_node = term.getTerm()->as_nearest_neighbor_query_node(); if (nn_node != nullptr) { - auto& raw_score = nn_node->get_raw_score(); + auto raw_score = nn_node->get_raw_score(); if (raw_score.has_value()) { auto& qtd = static_cast<QueryTermData &>(term.getTerm()->getQueryItem()); auto& td = qtd.getTermData(); diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h index c541f62646e..c74a2d1e3ee 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h @@ -65,7 +65,7 @@ public: void unpackMatchData(uint32_t docId); static void unpack_match_data(uint32_t docid, search::fef::MatchData& matchData, QueryWrapper& query); void runRankProgram(uint32_t docId); - search::FeatureSet::SP calculateFeatureSet(); + vespalib::FeatureSet::SP calculateFeatureSet(); void fillSearchResult(vdslib::SearchResult & searchResult); const search::fef::MatchData &getMatchData() const { return *_match_data; } void setRankScore(double score) { _score = score; } diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 7dc0c05cfaa..8980bc1f54d 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -664,14 +664,14 @@ SearchVisitor::RankController::onCompletedVisiting(vsm::GetDocsumsStateCallback // calculate summary features and set them on the callback object if (!_rankSetup->getSummaryFeatures().empty()) { LOG(debug, "Calculate summary features"); - search::FeatureSet::SP sf = _rankProcessor->calculateFeatureSet(); + vespalib::FeatureSet::SP sf = _rankProcessor->calculateFeatureSet(); docsumsStateCallback.setSummaryFeatures(sf); } // calculate rank features and set them on the callback object if (_dumpFeatures) { LOG(debug, "Calculate rank features"); - search::FeatureSet::SP rf = _dumpProcessor->calculateFeatureSet(); + vespalib::FeatureSet::SP rf = _dumpProcessor->calculateFeatureSet(); docsumsStateCallback.setRankFeatures(rf); } } diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 162c031ac9e..80df69f756e 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -38,7 +38,7 @@ namespace streaming { * @class storage::SearchVisitor * * @brief Visitor that applies a search query to visitor data and - * converts them to a SearchResultCommand and a DocumentSummaryCommand. + * converts them to a QueryResultCommand. **/ class SearchVisitor : public storage::Visitor { public: diff --git a/streamingvisitors/src/vespa/vsm/common/document.h b/streamingvisitors/src/vespa/vsm/common/document.h index de9ab052aa1..365d0e33ed0 100644 --- a/streamingvisitors/src/vespa/vsm/common/document.h +++ b/streamingvisitors/src/vespa/vsm/common/document.h @@ -13,7 +13,7 @@ namespace vespalib { namespace vsm { /// Type to identify fields in documents. -using FieldIdT = unsigned int; +using FieldIdT = uint32_t; /// A type to represent a list of FieldIds. using FieldIdTList = std::vector<FieldIdT>; /// A type to represent all the fields contained in all the indexs. diff --git a/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.cpp index cbf8903caab..9a89d0bebae 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.cpp @@ -61,13 +61,12 @@ void FieldSearcherBase::prepare(const QueryTermList & qtl) _qtlFastBuffer.resize(sizeof(*_qtlFast)*(_qtl.size()+1), 0x13); _qtlFast = reinterpret_cast<v16qi *>(reinterpret_cast<unsigned long>(&_qtlFastBuffer[0]+15) & ~0xf); _qtlFastSize = 0; - for(QueryTermList::iterator it=_qtl.begin(), mt=_qtl.end(); it != mt; it++) { - const QueryTerm & qt = **it; - memcpy(&_qtlFast[_qtlFastSize++], qt.getTerm(), std::min(size_t(16), qt.termLen())); + for (auto qt : _qtl) { + memcpy(&_qtlFast[_qtlFastSize++], qt->getTerm(), std::min(size_t(16), qt->termLen())); } } -FieldSearcher::FieldSearcher(const FieldIdT & fId, bool defaultPrefix) : +FieldSearcher::FieldSearcher(FieldIdT fId, bool defaultPrefix) : FieldSearcherBase(), _field(fId), _matchType(defaultPrefix ? PREFIX : REGULAR), @@ -89,16 +88,14 @@ FieldSearcher::~FieldSearcher() = default; bool FieldSearcher::search(const StorageDocument & doc) { - for(QueryTermList::iterator it=_qtl.begin(), mt=_qtl.end(); it != mt; it++) { - QueryTerm & qt = **it; - QueryTerm::FieldInfo & fInfo = qt.getFieldInfo(field()); - fInfo.setHitOffset(qt.getHitList().size()); + for (auto qt : _qtl) { + QueryTerm::FieldInfo & fInfo = qt->getFieldInfo(field()); + fInfo.setHitOffset(qt->getHitList().size()); } onSearch(doc); - for(QueryTermList::iterator it=_qtl.begin(), mt=_qtl.end(); it != mt; it++) { - QueryTerm & qt = **it; - QueryTerm::FieldInfo & fInfo = qt.getFieldInfo(field()); - fInfo.setHitCount(qt.getHitList().size() - fInfo.getHitOffset()); + for(auto qt : _qtl) { + QueryTerm::FieldInfo & fInfo = qt->getFieldInfo(field()); + fInfo.setHitCount(qt->getHitList().size() - fInfo.getHitOffset()); fInfo.setFieldLength(_words); } _words = 0; @@ -132,9 +129,8 @@ size_t FieldSearcher::countWords(const FieldRef & f) void FieldSearcher::prepareFieldId() { - for(QueryTermList::iterator it=_qtl.begin(), mt=_qtl.end(); it != mt; it++) { - QueryTerm & qt = **it; - qt.resizeFieldId(field()); + for(auto qt : _qtl) { + qt->resizeFieldId(field()); } } @@ -232,26 +228,26 @@ void FieldIdTSearcherMap::prepare(const DocumentTypeIndexFieldMapT& difm, QueryTermList qtl; query.getLeafs(qtl); vespalib::string tmp; - for (FieldIdTSearcherMap::iterator it = begin(), mt = end(); it != mt; it++) { + for (auto& searcher : *this) { QueryTermList onlyInIndex; - FieldIdT fid = (*it)->field(); - for (QueryTermList::iterator qt = qtl.begin(), mqt = qtl.end(); qt != mqt; qt++) { - QueryTerm * q = *qt; - for (DocumentTypeIndexFieldMapT::const_iterator dt(difm.begin()), dmt(difm.end()); dt != dmt; dt++) { - const IndexFieldMapT & fim = dt->second; - IndexFieldMapT::const_iterator found = fim.find(FieldSearchSpecMap::stripNonFields(q->index())); + FieldIdT fid = searcher->field(); + for (auto qt : qtl) { + for (const auto& doc_type_elem : difm) { + const IndexFieldMapT & fim = doc_type_elem.second; + auto found = fim.find(FieldSearchSpecMap::stripNonFields(qt->index())); if (found != fim.end()) { const FieldIdTList & index = found->second; - if ((find(index.begin(), index.end(), fid) != index.end()) && (find(onlyInIndex.begin(), onlyInIndex.end(), q) == onlyInIndex.end())) { - onlyInIndex.push_back(q); + if ((find(index.begin(), index.end(), fid) != index.end()) && (find(onlyInIndex.begin(), onlyInIndex.end(), qt) == onlyInIndex.end())) { + onlyInIndex.push_back(qt); } } else { - LOG(debug, "Could not find the requested index=%s in the index config map. Query does not fit search definition.", q->index().c_str()); + LOG(debug, "Could not find the requested index=%s in the index config map. Query does not fit search definition.", + qt->index().c_str()); } } } /// Should perhaps do a unique on onlyInIndex - (*it)->prepare(onlyInIndex, searcherBuf, field_paths, query_env); + searcher->prepare(onlyInIndex, searcherBuf, field_paths, query_env); if (LOG_WOULD_LOG(spam)) { char tmpBuf[16]; snprintf(tmpBuf, sizeof(tmpBuf), "%d", fid); diff --git a/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.h index 879902ca514..abc2bc9d870 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.h @@ -52,7 +52,7 @@ public: EXACT }; - FieldSearcher(const FieldIdT & fId, bool defaultPrefix=false); + FieldSearcher(FieldIdT fId, bool defaultPrefix=false); ~FieldSearcher() override; virtual std::unique_ptr<FieldSearcher> duplicate() const = 0; bool search(const StorageDocument & doc); @@ -61,8 +61,8 @@ public: const vsm::FieldPathMapT& field_paths, search::fef::IQueryEnvironment& query_env); - const FieldIdT & field() const { return _field; } - void field(const FieldIdT & v) { _field = v; prepareFieldId(); } + FieldIdT field() const { return _field; } + void field(FieldIdT v) { _field = v; prepareFieldId(); } bool prefix() const { return _matchType == PREFIX; } bool substring() const { return _matchType == SUBSTRING; } bool suffix() const { return _matchType == SUFFIX; } diff --git a/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp index 8585975ca3c..578fc9fe0e5 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp @@ -36,8 +36,7 @@ void FloatFieldSearcherT<T>::prepare(search::streaming::QueryTermList& qtl, { _floatTerm.clear(); FieldSearcher::prepare(qtl, buf, field_paths, query_env); - for (QueryTermList::const_iterator it=qtl.begin(); it < qtl.end(); it++) { - const QueryTerm * qt = *it; + for (auto qt : qtl) { size_t sz(qt->termLen()); if (sz) { double low; diff --git a/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp index 6b0bbbb368d..43ecba29b33 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp @@ -35,7 +35,7 @@ void GeoPosFieldSearcher::prepare(search::streaming::QueryTermList& qtl, { _geoPosTerm.clear(); FieldSearcher::prepare(qtl, buf, field_paths, query_env); - for (const QueryTerm * qt : qtl) { + for (auto qt : qtl) { const vespalib::string & str = qt->getTermString(); GeoLocationParser parser; bool valid = parser.parseNoField(str); diff --git a/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.cpp index 18b286946f7..0fb71a3c3c6 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.cpp @@ -26,8 +26,7 @@ void IntFieldSearcher::prepare(search::streaming::QueryTermList& qtl, { _intTerm.clear(); FieldSearcher::prepare(qtl, buf, field_paths, query_env); - for (QueryTermList::const_iterator it=qtl.begin(); it < qtl.end(); it++) { - const QueryTerm * qt = *it; + for (auto qt : qtl) { size_t sz(qt->termLen()); if (sz) { int64_t low; diff --git a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp index 045ec9b04a3..db4ee12438e 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp @@ -48,11 +48,20 @@ NearestNeighborFieldSearcher::NodeAndCalc::NodeAndCalc(search::streaming::Neares std::unique_ptr<search::tensor::DistanceCalculator> calc_in) : node(node_in), calc(std::move(calc_in)), - distance_threshold(calc->function().convert_threshold(node->get_distance_threshold())) + heap(node->get_target_hits()) { + node->set_raw_score_calc(this); + heap.set_distance_threshold(calc->function().convert_threshold(node->get_distance_threshold())); } -NearestNeighborFieldSearcher::NearestNeighborFieldSearcher(const FieldIdT& fid, +double +NearestNeighborFieldSearcher::NodeAndCalc::to_raw_score(double distance) +{ + heap.used(distance); + return calc->function().to_rawscore(distance); +} + +NearestNeighborFieldSearcher::NearestNeighborFieldSearcher(FieldIdT fid, search::attribute::DistanceMetric metric) : FieldSearcher(fid), _metric(metric), @@ -100,7 +109,7 @@ NearestNeighborFieldSearcher::prepare(search::streaming::QueryTermList& qtl, } try { auto calc = DistanceCalculator::make_with_validation(*_attr, *tensor_value); - _calcs.emplace_back(nn_term, std::move(calc)); + _calcs.push_back(std::make_unique<NodeAndCalc>(nn_term, std::move(calc))); } catch (const vespalib::IllegalArgumentException& ex) { vespalib::Issue::report("Could not create DistanceCalculator for NearestNeighborQueryNode(%s, %s): %s", nn_term->index().c_str(), nn_term->get_query_tensor_name().c_str(), ex.what()); @@ -116,10 +125,10 @@ NearestNeighborFieldSearcher::onValue(const document::FieldValue& fv) if (tfv && tfv->getAsTensorPtr()) { _attr->add(*tfv->getAsTensorPtr(), 1); for (auto& elem : _calcs) { - double distance = elem.calc->calc_with_limit(scratch_docid, elem.distance_threshold); - if (distance <= elem.distance_threshold) { - double score = elem.calc->function().to_rawscore(distance); - elem.node->set_raw_score(score); + double distance_limit = elem->heap.distanceLimit(); + double distance = elem->calc->calc_with_limit(scratch_docid, distance_limit); + if (distance <= distance_limit) { + elem->node->set_distance(distance); } } } diff --git a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.h b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.h index 83f2c444e5a..d5d751cd637 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.h @@ -5,6 +5,8 @@ #include "fieldsearcher.h" #include <vespa/eval/eval/value_type.h> #include <vespa/searchcommon/attribute/distance_metric.h> +#include <vespa/searchlib/query/streaming/nearest_neighbor_query_node.h> +#include <vespa/searchlib/queryeval/nearest_neighbor_distance_heap.h> #include <vespa/searchlib/tensor/distance_calculator.h> #include <vespa/searchlib/tensor/tensor_ext_attribute.h> @@ -14,8 +16,6 @@ namespace search::tensor { class TensorExtAttribute; } -namespace search::streaming { class NearestNeighborQueryNode; } - namespace vsm { /** @@ -26,19 +26,22 @@ namespace vsm { */ class NearestNeighborFieldSearcher : public FieldSearcher { private: - struct NodeAndCalc { + class NodeAndCalc : search::streaming::NearestNeighborQueryNode::RawScoreCalculator { + public: search::streaming::NearestNeighborQueryNode* node; std::unique_ptr<search::tensor::DistanceCalculator> calc; - double distance_threshold; + search::queryeval::NearestNeighborDistanceHeap heap; NodeAndCalc(search::streaming::NearestNeighborQueryNode* node_in, std::unique_ptr<search::tensor::DistanceCalculator> calc_in); + + double to_raw_score(double distance) override; }; search::attribute::DistanceMetric _metric; std::unique_ptr<search::tensor::TensorExtAttribute> _attr; - std::vector<NodeAndCalc> _calcs; + std::vector<std::unique_ptr<NodeAndCalc>> _calcs; public: - NearestNeighborFieldSearcher(const FieldIdT& fid, + NearestNeighborFieldSearcher(FieldIdT fid, search::attribute::DistanceMetric metric); ~NearestNeighborFieldSearcher(); diff --git a/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.cpp index f15290526d9..6a46e4604be 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.cpp @@ -34,10 +34,9 @@ bool StrChrFieldSearcher::matchDoc(const FieldRef & fieldRef) _words += countWords(fieldRef); } } else { - for(QueryTermList::iterator it=_qtl.begin(), mt=_qtl.end(); it != mt; it++) { - QueryTerm & qt = **it; - if (fieldRef.size() >= qt.termLen()) { - _words += matchTerm(fieldRef, qt); + for (auto qt : _qtl) { + if (fieldRef.size() >= qt->termLen()) { + _words += matchTerm(fieldRef, *qt); } else { _words += countWords(fieldRef); } @@ -49,7 +48,7 @@ bool StrChrFieldSearcher::matchDoc(const FieldRef & fieldRef) size_t StrChrFieldSearcher::shortestTerm() const { size_t mintsz(_qtl.front()->termLen()); - for(QueryTermList::const_iterator it=_qtl.begin()+1, mt=_qtl.end(); it != mt; it++) { + for (auto it=_qtl.begin()+1, mt=_qtl.end(); it != mt; it++) { const QueryTerm & qt = **it; mintsz = std::min(mintsz, qt.termLen()); } diff --git a/streamingvisitors/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.cpp index 977602a691c..a7ad02fa9d9 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.cpp @@ -17,9 +17,8 @@ size_t UTF8ExactStringFieldSearcher::matchTerms(const FieldRef & f, const size_t mintsz) { (void) mintsz; - for (QueryTermList::iterator it = _qtl.begin(), mt = _qtl.end(); it != mt; ++it) { - QueryTerm & qt = **it; - matchTermExact(f, qt); + for (auto qt : _qtl) { + matchTermExact(f, *qt); } return 1; } diff --git a/streamingvisitors/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.cpp index 9aef99f9fa1..5809738456f 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.cpp @@ -20,8 +20,8 @@ UTF8FlexibleStringFieldSearcher::matchTerms(const FieldRef & f, const size_t min { (void) mintsz; size_t words = 0; - for (QueryTermList::iterator it = _qtl.begin(); it != _qtl.end(); ++it) { - words = matchTerm(f, **it); + for (auto qt : _qtl) { + words = matchTerm(f, *qt); } return words; } diff --git a/streamingvisitors/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp index 0d93009655c..e8ac87b836b 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp @@ -29,15 +29,14 @@ UTF8StrChrFieldSearcher::matchTerms(const FieldRef & f, const size_t mintsz) for( ; n < e; ) { if (!*n) { _zeroCount++; n++; } n = tokenize(n, _buf->capacity(), fn, fl); - for(QueryTermList::iterator it=_qtl.begin(), mt=_qtl.end(); it != mt; it++) { - QueryTerm & qt = **it; + for (auto qt : _qtl) { const cmptype_t * term; - termsize_t tsz = qt.term(term); - if ((tsz <= fl) && (prefix() || qt.isPrefix() || (tsz == fl))) { + termsize_t tsz = qt->term(term); + if ((tsz <= fl) && (prefix() || qt->isPrefix() || (tsz == fl))) { const cmptype_t *tt=term, *et=term+tsz; for (const cmptype_t *fnt=fn; (tt < et) && (*tt == *fnt); tt++, fnt++); if (tt == et) { - addHit(qt, words); + addHit(*qt, words); } } } diff --git a/streamingvisitors/src/vespa/vsm/searcher/utf8substringsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsearcher.cpp index fd327d3a3df..adcf7a937c1 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/utf8substringsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsearcher.cpp @@ -29,15 +29,14 @@ UTF8SubStringFieldSearcher::matchTerms(const FieldRef & f, const size_t mintsz) const cmptype_t * fre = fe - mintsz; termcount_t words(0); for(words = 0; fn <= fre; ) { - for(QueryTermList::iterator it=_qtl.begin(), mt=_qtl.end(); it != mt; it++) { - QueryTerm & qt = **it; + for (auto qt : _qtl) { const cmptype_t * term; - termsize_t tsz = qt.term(term); + termsize_t tsz = qt->term(term); const cmptype_t *tt=term, *et=term+tsz, *fnt=fn; for (; (tt < et) && (*tt == *fnt); tt++, fnt++); if (tt == et) { - addHit(qt, words); + addHit(*qt, words); } } if ( ! Fast_UnicodeUtil::IsWordChar(*fn++) ) { diff --git a/streamingvisitors/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp index 9046c0063d5..89388c01354 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp @@ -41,10 +41,9 @@ UTF8SubstringSnippetModifier::matchTerms(const FieldRef & f, const size_t mintsz const cmptype_t * drend = dend - mintsz; termcount_t words = 0; for(; ditr <= drend; ) { - for (QueryTermList::iterator itr = _qtl.begin(); itr != _qtl.end(); ++itr) { - QueryTerm & qt = **itr; + for (auto qt : _qtl) { const cmptype_t * term; - termsize_t tsz = qt.term(term); + termsize_t tsz = qt->term(term); const cmptype_t * titr = term; const cmptype_t * tend = term + tsz; @@ -58,7 +57,7 @@ UTF8SubstringSnippetModifier::matchTerms(const FieldRef & f, const size_t mintsz // If we have overlapping matches only the first one will be considered. insertSeparators(mbegin, mend); } - addHit(qt, words); + addHit(*qt, words); } } if ( ! Fast_UnicodeUtil::IsWordChar(*ditr++) ) { diff --git a/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.cpp b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.cpp index 7043e63ec87..98ed8a26938 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.cpp @@ -1,17 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "fieldsearchspec.h" +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vsm/searcher/boolfieldsearcher.h> +#include <vespa/vsm/searcher/floatfieldsearcher.h> +#include <vespa/vsm/searcher/futf8strchrfieldsearcher.h> +#include <vespa/vsm/searcher/geo_pos_field_searcher.h> +#include <vespa/vsm/searcher/intfieldsearcher.h> +#include <vespa/vsm/searcher/nearest_neighbor_field_searcher.h> +#include <vespa/vsm/searcher/utf8exactstringfieldsearcher.h> #include <vespa/vsm/searcher/utf8flexiblestringfieldsearcher.h> #include <vespa/vsm/searcher/utf8strchrfieldsearcher.h> #include <vespa/vsm/searcher/utf8substringsearcher.h> #include <vespa/vsm/searcher/utf8suffixstringfieldsearcher.h> -#include <vespa/vsm/searcher/utf8exactstringfieldsearcher.h> -#include <vespa/vsm/searcher/futf8strchrfieldsearcher.h> -#include <vespa/vsm/searcher/intfieldsearcher.h> -#include <vespa/vsm/searcher/boolfieldsearcher.h> -#include <vespa/vsm/searcher/floatfieldsearcher.h> -#include <vespa/vsm/searcher/geo_pos_field_searcher.h> -#include <vespa/vespalib/stllike/asciistream.h> #include <regex> #include <vespa/log/log.h> @@ -109,6 +110,10 @@ FieldSearchSpec::FieldSearchSpec(const FieldIdT & fid, const vespalib::string & case VsmfieldsConfig::Fieldspec::Searchmethod::GEOPOS: _searcher = std::make_unique<GeoPosFieldSearcher>(fid); break; + case VsmfieldsConfig::Fieldspec::Searchmethod::NEAREST_NEIGHBOR: + auto dm = NearestNeighborFieldSearcher::distance_metric_from_string(arg1); + _searcher = std::make_unique<NearestNeighborFieldSearcher>(fid, dm); + break; } if (_searcher) { setMatchType(_searcher, arg1); diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h index 77ed9573e54..ba87ccfef05 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h @@ -5,11 +5,11 @@ #include <vespa/searchlib/query/base.h> #include <vespa/vsm/config/vsm-cfif.h> #include <vespa/config-summary.h> -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchsummary/docsummary/docsumwriter.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> #include <vespa/juniper/rpinterface.h> +#include <vespa/vespalib/util/featureset.h> using search::docsummary::ResultConfig; using search::docsummary::ResultClass; @@ -28,8 +28,8 @@ class IMatchingElementsFiller; class GetDocsumsStateCallback : public search::docsummary::GetDocsumsStateCallback { private: - search::FeatureSet::SP _summaryFeatures; - search::FeatureSet::SP _rankFeatures; + vespalib::FeatureSet::SP _summaryFeatures; + vespalib::FeatureSet::SP _rankFeatures; std::unique_ptr<IMatchingElementsFiller> _matching_elements_filler; public: @@ -37,8 +37,8 @@ public: void fillSummaryFeatures(GetDocsumsState& state) override; void fillRankFeatures(GetDocsumsState& state) override; std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields& fields) override; - void setSummaryFeatures(const search::FeatureSet::SP & sf) { _summaryFeatures = sf; } - void setRankFeatures(const search::FeatureSet::SP & rf) { _rankFeatures = rf; } + void setSummaryFeatures(const vespalib::FeatureSet::SP & sf) { _summaryFeatures = sf; } + void setRankFeatures(const vespalib::FeatureSet::SP & rf) { _rankFeatures = rf; } void set_matching_elements_filler(std::unique_ptr<IMatchingElementsFiller> matching_elements_filler); ~GetDocsumsStateCallback() override; }; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java new file mode 100644 index 00000000000..c2ab22f4921 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api; + +public record DefaultSignedIdentityDocument(String signature, int signingKeyVersion, int documentVersion, + String data, IdentityDocument identityDocument) implements SignedIdentityDocument { + + public DefaultSignedIdentityDocument { + identityDocument = EntityBindingsMapper.fromIdentityDocumentData(data); + } + + public DefaultSignedIdentityDocument(String signature, int signingKeyVersion, int documentVersion, String data) { + this(signature,signingKeyVersion,documentVersion, data, null); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index 2d77d2ceda1..a695e10a29c 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -6,8 +6,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.identityprovider.api.bindings.DefaultSignedIdentityDocumentEntity; +import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentEntity; +import com.yahoo.vespa.athenz.identityprovider.api.bindings.LegacySignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.utils.AthenzIdentities; +import com.yahoo.yolean.Exceptions; import java.io.IOException; import java.io.InputStream; @@ -16,6 +20,8 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.Base64; import java.util.Optional; import static com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId.fromDottedString; @@ -24,6 +30,7 @@ import static com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId. * Utility class for mapping objects model types and their Jackson binding versions. * * @author bjorncs + * @author mortent */ public class EntityBindingsMapper { @@ -48,39 +55,60 @@ public class EntityBindingsMapper { } public static SignedIdentityDocument toSignedIdentityDocument(SignedIdentityDocumentEntity entity) { - return new SignedIdentityDocument( - entity.signature(), - entity.signingKeyVersion(), - fromDottedString(entity.providerUniqueId()), - new AthenzService(entity.providerService()), - entity.documentVersion(), - entity.configServerHostname(), - entity.instanceHostname(), - entity.createdAt(), - entity.ipAddresses(), - IdentityType.fromId(entity.identityType()), - Optional.ofNullable(entity.clusterType()).map(ClusterType::from).orElse(null), - entity.ztsUrl(), - Optional.ofNullable(entity.serviceIdentity()).map(AthenzIdentities::from).orElse(null), - entity.unknownAttributes()); + if (entity instanceof LegacySignedIdentityDocumentEntity docEntity) { + IdentityDocument doc = new IdentityDocument( + fromDottedString(docEntity.providerUniqueId()), + new AthenzService(docEntity.providerService()), + docEntity.configServerHostname(), + docEntity.instanceHostname(), + docEntity.createdAt(), + docEntity.ipAddresses(), + IdentityType.fromId(docEntity.identityType()), + Optional.ofNullable(docEntity.clusterType()).map(ClusterType::from).orElse(null), + docEntity.ztsUrl(), + Optional.ofNullable(docEntity.serviceIdentity()).map(AthenzIdentities::from).orElse(null), + docEntity.unknownAttributes()); + return new LegacySignedIdentityDocument( + docEntity.signature(), + docEntity.signingKeyVersion(), + entity.documentVersion(), + doc); + } else if (entity instanceof DefaultSignedIdentityDocumentEntity docEntity) { + return new DefaultSignedIdentityDocument(docEntity.signature(), + docEntity.signingKeyVersion(), + docEntity.documentVersion(), + docEntity.data()); + } else { + throw new IllegalArgumentException("Unknown signed identity document type: " + entity.getClass().getName()); + } } public static SignedIdentityDocumentEntity toSignedIdentityDocumentEntity(SignedIdentityDocument model) { - return new SignedIdentityDocumentEntity( - model.signature(), - model.signingKeyVersion(), - model.providerUniqueId().asDottedString(), - model.providerService().getFullName(), - model.documentVersion(), - model.configServerHostname(), - model.instanceHostname(), - model.createdAt(), - model.ipAddresses(), - model.identityType().id(), - Optional.ofNullable(model.clusterType()).map(ClusterType::toConfigValue).orElse(null), - model.ztsUrl(), - Optional.ofNullable(model.serviceIdentity()).map(AthenzIdentity::getFullName).orElse(null), - model.unknownAttributes()); + if (model instanceof LegacySignedIdentityDocument legacyModel) { + IdentityDocument idDoc = legacyModel.identityDocument(); + return new LegacySignedIdentityDocumentEntity( + legacyModel.signature(), + legacyModel.signingKeyVersion(), + idDoc.providerUniqueId().asDottedString(), + idDoc.providerService().getFullName(), + legacyModel.documentVersion(), + idDoc.configServerHostname(), + idDoc.instanceHostname(), + idDoc.createdAt(), + idDoc.ipAddresses(), + idDoc.identityType().id(), + Optional.ofNullable(idDoc.clusterType()).map(ClusterType::toConfigValue).orElse(null), + idDoc.ztsUrl(), + Optional.ofNullable(idDoc.serviceIdentity()).map(AthenzIdentity::getFullName).orElse(null), + idDoc.unknownAttributes()); + } else if (model instanceof DefaultSignedIdentityDocument defaultModel){ + return new DefaultSignedIdentityDocumentEntity(defaultModel.signature(), + defaultModel.signingKeyVersion(), + defaultModel.documentVersion(), + defaultModel.data()); + } else { + throw new IllegalArgumentException("Unsupported model type: " + model.getClass().getName()); + } } public static SignedIdentityDocument readSignedIdentityDocumentFromFile(Path file) { @@ -104,4 +132,40 @@ public class EntityBindingsMapper { } } + public static IdentityDocument fromIdentityDocumentData(String data) { + byte[] decoded = Base64.getDecoder().decode(data); + IdentityDocumentEntity docEntity = Exceptions.uncheck(() -> mapper.readValue(decoded, IdentityDocumentEntity.class)); + return new IdentityDocument( + fromDottedString(docEntity.providerUniqueId()), + new AthenzService(docEntity.providerService()), + docEntity.configServerHostname(), + docEntity.instanceHostname(), + docEntity.createdAt(), + docEntity.ipAddresses(), + IdentityType.fromId(docEntity.identityType()), + Optional.ofNullable(docEntity.clusterType()).map(ClusterType::from).orElse(null), + docEntity.ztsUrl(), + Optional.ofNullable(docEntity.serviceIdentity()).map(AthenzIdentities::from).orElse(null), + docEntity.unknownAttributes()); + } + + public static String toIdentityDocmentData(IdentityDocument identityDocument) { + IdentityDocumentEntity documentEntity = new IdentityDocumentEntity( + identityDocument.providerUniqueId().asDottedString(), + identityDocument.providerService().getFullName(), + identityDocument.configServerHostname(), + identityDocument.instanceHostname(), + identityDocument.createdAt(), + identityDocument.ipAddresses(), + identityDocument.identityType().id(), + Optional.ofNullable(identityDocument.clusterType()).map(ClusterType::toConfigValue).orElse(null), + identityDocument.ztsUrl(), + identityDocument.serviceIdentity().getFullName()); + try { + byte[] bytes = mapper.writeValueAsBytes(documentEntity); + return Base64.getEncoder().encodeToString(bytes); + } catch (JsonProcessingException e) { + throw new RuntimeException("Error during serialization of identity document.", e); + } + } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java new file mode 100644 index 00000000000..577584db185 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api; + +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzService; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Represents an unsigned identity document + * @author mortent + */ +public record IdentityDocument(VespaUniqueInstanceId providerUniqueId, AthenzService providerService, String configServerHostname, + String instanceHostname, Instant createdAt, Set<String> ipAddresses, + IdentityType identityType, ClusterType clusterType, String ztsUrl, + AthenzIdentity serviceIdentity, Map<String, Object> unknownAttributes) { + + public IdentityDocument { + ipAddresses = Set.copyOf(ipAddresses); + + Map<String, Object> nonNull = new HashMap<>(); + unknownAttributes.forEach((key, value) -> { + if (value != null) nonNull.put(key, value); + }); + // Map.copyOf() does not allow null values + unknownAttributes = Map.copyOf(nonNull); + } + + public IdentityDocument(VespaUniqueInstanceId providerUniqueId, AthenzService providerService, String configServerHostname, + String instanceHostname, Instant createdAt, Set<String> ipAddresses, + IdentityType identityType, ClusterType clusterType, String ztsUrl, + AthenzIdentity serviceIdentity) { + this(providerUniqueId, providerService, configServerHostname, instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, Map.of()); + } + + + public IdentityDocument withServiceIdentity(AthenzService athenzService) { + return new IdentityDocument( + this.providerUniqueId, + this.providerService, + this.configServerHostname, + this.instanceHostname, + this.createdAt, + this.ipAddresses, + this.identityType, + this.clusterType, + this.ztsUrl, + athenzService, + this.unknownAttributes); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocumentClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocumentClient.java index 5a0f77ec765..0e13cba8de9 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocumentClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocumentClient.java @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.api; +import java.util.OptionalInt; + /** * A client that communicates that fetches an identity document. * * @author bjorncs */ public interface IdentityDocumentClient { - SignedIdentityDocument getNodeIdentityDocument(String host); - SignedIdentityDocument getTenantIdentityDocument(String host); + SignedIdentityDocument getNodeIdentityDocument(String host, int documentVersion); + SignedIdentityDocument getTenantIdentityDocument(String host, int documentVersion); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/LegacySignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/LegacySignedIdentityDocument.java new file mode 100644 index 00000000000..220bc72a017 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/LegacySignedIdentityDocument.java @@ -0,0 +1,6 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api; + +public record LegacySignedIdentityDocument(String signature, int signingKeyVersion, int documentVersion, + IdentityDocument identityDocument) implements SignedIdentityDocument { +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index de78d81cd1b..4e3bd8dee91 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -1,54 +1,20 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.api; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; - -import java.net.URL; -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - /** * A signed identity document. - * The {@link #unknownAttributes()} member provides forward compatibility and ensures any new/unknown fields are kept intact when serialized to JSON. - * * @author bjorncs + * @author mortent */ -public record SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, - AthenzService providerService, int documentVersion, String configServerHostname, - String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType, String ztsUrl, - AthenzIdentity serviceIdentity, Map<String, Object> unknownAttributes) { - - public SignedIdentityDocument { - ipAddresses = Set.copyOf(ipAddresses); - - Map<String, Object> nonNull = new HashMap<>(); - unknownAttributes.forEach((key, value) -> { - if (value != null) nonNull.put(key, value); - }); - // Map.copyOf() does not allow null values - unknownAttributes = Map.copyOf(nonNull); - } - - public SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, - AthenzService providerService, int documentVersion, String configServerHostname, - String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType, String ztsUrl, AthenzIdentity serviceIdentity) { - this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, Map.of()); - } - - public static final int DEFAULT_DOCUMENT_VERSION = 3; - - public boolean outdated() { return documentVersion < DEFAULT_DOCUMENT_VERSION; } +public interface SignedIdentityDocument { - public SignedIdentityDocument withServiceIdentity(AthenzIdentity identity) { - return new SignedIdentityDocument(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, instanceHostname, createdAt, - ipAddresses, identityType, clusterType, ztsUrl, identity); - } + int LEGACY_DEFAULT_DOCUMENT_VERSION = 3; + int DEFAULT_DOCUMENT_VERSION = 4; + default boolean outdated() { return documentVersion() < LEGACY_DEFAULT_DOCUMENT_VERSION; } + IdentityDocument identityDocument(); + String signature(); + int signingKeyVersion(); + int documentVersion(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/DefaultSignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/DefaultSignedIdentityDocumentEntity.java new file mode 100644 index 00000000000..3aaff011415 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/DefaultSignedIdentityDocumentEntity.java @@ -0,0 +1,12 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api.bindings; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record DefaultSignedIdentityDocumentEntity( + @JsonProperty("signature") String signature, + @JsonProperty("signing-key-version") int signingKeyVersion, + @JsonProperty("document-version") int documentVersion, + @JsonProperty("data") String data) + implements SignedIdentityDocumentEntity { +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java new file mode 100644 index 00000000000..946eacc67eb --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api.bindings; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * @author bjorncs + * @author mortent + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record IdentityDocumentEntity(String providerUniqueId, String providerService, + String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, + String identityType, String clusterType, String ztsUrl, String serviceIdentity, Map<String, Object> unknownAttributes) { + + @JsonCreator + public IdentityDocumentEntity(@JsonProperty("provider-unique-id") String providerUniqueId, + @JsonProperty("provider-service") String providerService, + @JsonProperty("configserver-hostname") String configServerHostname, + @JsonProperty("instance-hostname") String instanceHostname, + @JsonProperty("created-at") Instant createdAt, + @JsonProperty("ip-addresses") Set<String> ipAddresses, + @JsonProperty("identity-type") String identityType, + @JsonProperty("cluster-type") String clusterType, + @JsonProperty("zts-url") String ztsUrl, + @JsonProperty("service-identity") String serviceIdentity) { + this(providerUniqueId, providerService, configServerHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, new HashMap<>()); + } + + @JsonProperty("provider-unique-id") @Override public String providerUniqueId() { return providerUniqueId; } + @JsonProperty("provider-service") @Override public String providerService() { return providerService; } + @JsonProperty("configserver-hostname") @Override public String configServerHostname() { return configServerHostname; } + @JsonProperty("instance-hostname") @Override public String instanceHostname() { return instanceHostname; } + @JsonProperty("created-at") @Override public Instant createdAt() { return createdAt; } + @JsonProperty("ip-addresses") @Override public Set<String> ipAddresses() { return ipAddresses; } + @JsonProperty("identity-type") @Override public String identityType() { return identityType; } + @JsonProperty("cluster-type") @Override public String clusterType() { return clusterType; } + @JsonProperty("zts-url") @Override public String ztsUrl() { return ztsUrl; } + @JsonProperty("service-identity") @Override public String serviceIdentity() { return serviceIdentity; } + @JsonAnyGetter @Override public Map<String, Object> unknownAttributes() { return unknownAttributes; } + @JsonAnySetter public void set(String name, Object value) { unknownAttributes.put(name, value); } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/LegacySignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/LegacySignedIdentityDocumentEntity.java new file mode 100644 index 00000000000..e00ab9978f6 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/LegacySignedIdentityDocumentEntity.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api.bindings; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * @author bjorncs + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record LegacySignedIdentityDocumentEntity ( + String signature, int signingKeyVersion, String providerUniqueId, String providerService, int documentVersion, + String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, + String identityType, String clusterType, String ztsUrl, String serviceIdentity, Map<String, Object> unknownAttributes) implements SignedIdentityDocumentEntity { + + @JsonCreator + public LegacySignedIdentityDocumentEntity(@JsonProperty("signature") String signature, + @JsonProperty("signing-key-version") int signingKeyVersion, + @JsonProperty("provider-unique-id") String providerUniqueId, + @JsonProperty("provider-service") String providerService, + @JsonProperty("document-version") int documentVersion, + @JsonProperty("configserver-hostname") String configServerHostname, + @JsonProperty("instance-hostname") String instanceHostname, + @JsonProperty("created-at") Instant createdAt, + @JsonProperty("ip-addresses") Set<String> ipAddresses, + @JsonProperty("identity-type") String identityType, + @JsonProperty("cluster-type") String clusterType, + @JsonProperty("zts-url") String ztsUrl, + @JsonProperty("service-identity") String serviceIdentity) { + this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, new HashMap<>()); + } + + @JsonProperty("signature") @Override public String signature() { return signature; } + @JsonProperty("signing-key-version") @Override public int signingKeyVersion() { return signingKeyVersion; } + @JsonProperty("provider-unique-id") @Override public String providerUniqueId() { return providerUniqueId; } + @JsonProperty("provider-service") @Override public String providerService() { return providerService; } + @JsonProperty("document-version") @Override public int documentVersion() { return documentVersion; } + @JsonProperty("configserver-hostname") @Override public String configServerHostname() { return configServerHostname; } + @JsonProperty("instance-hostname") @Override public String instanceHostname() { return instanceHostname; } + @JsonProperty("created-at") @Override public Instant createdAt() { return createdAt; } + @JsonProperty("ip-addresses") @Override public Set<String> ipAddresses() { return ipAddresses; } + @JsonProperty("identity-type") @Override public String identityType() { return identityType; } + @JsonProperty("cluster-type") @Override public String clusterType() { return clusterType; } + @JsonProperty("zts-url") @Override public String ztsUrl() { return ztsUrl; } + @JsonProperty("service-identity") @Override public String serviceIdentity() { return serviceIdentity; } + @JsonAnyGetter @Override public Map<String, Object> unknownAttributes() { return unknownAttributes; } + @JsonAnySetter public void set(String name, Object value) { unknownAttributes.put(name, value); } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index fc0dff3b97b..174c76f7fa9 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -1,57 +1,77 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.api.bindings; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * @author bjorncs - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public record SignedIdentityDocumentEntity( - String signature, int signingKeyVersion, String providerUniqueId, String providerService, int documentVersion, - String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - String identityType, String clusterType, String ztsUrl, String serviceIdentity, Map<String, Object> unknownAttributes) { - - @JsonCreator - public SignedIdentityDocumentEntity(@JsonProperty("signature") String signature, - @JsonProperty("signing-key-version") int signingKeyVersion, - @JsonProperty("provider-unique-id") String providerUniqueId, - @JsonProperty("provider-service") String providerService, - @JsonProperty("document-version") int documentVersion, - @JsonProperty("configserver-hostname") String configServerHostname, - @JsonProperty("instance-hostname") String instanceHostname, - @JsonProperty("created-at") Instant createdAt, - @JsonProperty("ip-addresses") Set<String> ipAddresses, - @JsonProperty("identity-type") String identityType, - @JsonProperty("cluster-type") String clusterType, - @JsonProperty("zts-url") String ztsUrl, - @JsonProperty("service-identity") String serviceIdentity) { - this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, new HashMap<>()); - } - @JsonProperty("signature") @Override public String signature() { return signature; } - @JsonProperty("signing-key-version") @Override public int signingKeyVersion() { return signingKeyVersion; } - @JsonProperty("provider-unique-id") @Override public String providerUniqueId() { return providerUniqueId; } - @JsonProperty("provider-service") @Override public String providerService() { return providerService; } - @JsonProperty("document-version") @Override public int documentVersion() { return documentVersion; } - @JsonProperty("configserver-hostname") @Override public String configServerHostname() { return configServerHostname; } - @JsonProperty("instance-hostname") @Override public String instanceHostname() { return instanceHostname; } - @JsonProperty("created-at") @Override public Instant createdAt() { return createdAt; } - @JsonProperty("ip-addresses") @Override public Set<String> ipAddresses() { return ipAddresses; } - @JsonProperty("identity-type") @Override public String identityType() { return identityType; } - @JsonProperty("cluster-type") @Override public String clusterType() { return clusterType; } - @JsonProperty("zts-url") @Override public String ztsUrl() { return ztsUrl; } - @JsonProperty("service-identity") @Override public String serviceIdentity() { return serviceIdentity; } - @JsonAnyGetter @Override public Map<String, Object> unknownAttributes() { return unknownAttributes; } - @JsonAnySetter public void set(String name, Object value) { unknownAttributes.put(name, value); } +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.DatabindContext; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.annotation.JsonTypeIdResolver; +import com.fasterxml.jackson.databind.jsontype.TypeIdResolver; +import com.fasterxml.jackson.databind.type.TypeFactory; +import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; + +import java.io.IOException; +import java.util.Objects; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "document-version", visible = true) +@JsonTypeIdResolver(SignedIdentityDocumentEntityTypeResolver.class) +public interface SignedIdentityDocumentEntity { + int documentVersion(); } + +class SignedIdentityDocumentEntityTypeResolver implements TypeIdResolver { + JavaType javaType; + + @Override + public void init(JavaType javaType) { + this.javaType = javaType; + } + + @Override + public String idFromValue(Object o) { + return idFromValueAndType(o, o.getClass()); + } + + @Override + public String idFromValueAndType(Object o, Class<?> aClass) { + if (Objects.isNull(o)) { + throw new IllegalArgumentException("Cannot serialize null oject"); + } else { + if (o instanceof SignedIdentityDocumentEntity s) { + return Integer.toString(s.documentVersion()); + } else { + throw new IllegalArgumentException("Cannot serialize class: " + o.getClass()); + } + } + } + + @Override + public String idFromBaseType() { + return idFromValueAndType(null, javaType.getRawClass()); + } + + @Override + public JavaType typeFromId(DatabindContext databindContext, String s) throws IOException { + try { + int version = Integer.parseInt(s); + Class<? extends SignedIdentityDocumentEntity> cls = version <= SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION + ? LegacySignedIdentityDocumentEntity.class + : DefaultSignedIdentityDocumentEntity.class; + return TypeFactory.defaultInstance().constructSpecializedType(javaType,cls); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Unable to deserialize document with version: \"%s\"".formatted(s)); + } + } + + @Override + public String getDescForKnownTypeIds() { + return "Type resolver for SignedIdentityDocumentEntity"; + } + + @Override + public JsonTypeInfo.Id getMechanism() { + return JsonTypeInfo.Id.CUSTOM; + } +}
\ No newline at end of file diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java index cc9d3b2be65..1858653c9b4 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; @@ -74,7 +75,9 @@ class AthenzCredentialsService { } KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); IdentityDocumentClient identityDocumentClient = createIdentityDocumentClient(); - SignedIdentityDocument document = identityDocumentClient.getTenantIdentityDocument(hostname); + // Use legacy version for now. + SignedIdentityDocument signedDocument = identityDocumentClient.getTenantIdentityDocument(hostname, SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION); + IdentityDocument document = signedDocument.identityDocument(); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( tenantIdentity, document.providerUniqueId(), @@ -87,16 +90,17 @@ class AthenzCredentialsService { ztsClient.registerInstance( configserverIdentity, tenantIdentity, - EntityBindingsMapper.toAttestationData(document), + EntityBindingsMapper.toAttestationData(signedDocument), csr); X509Certificate certificate = instanceIdentity.certificate(); - writeCredentialsToDisk(keyPair.getPrivate(), certificate, document); - return new AthenzCredentials(certificate, keyPair, document); + writeCredentialsToDisk(keyPair.getPrivate(), certificate, signedDocument); + return new AthenzCredentials(certificate, keyPair, signedDocument); } } - AthenzCredentials updateCredentials(SignedIdentityDocument document, SSLContext sslContext) { + AthenzCredentials updateCredentials(SignedIdentityDocument signedDocument, SSLContext sslContext) { KeyPair newKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); + IdentityDocument document = signedDocument.identityDocument(); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( tenantIdentity, document.providerUniqueId(), @@ -112,8 +116,8 @@ class AthenzCredentialsService { document.providerUniqueId().asDottedString(), csr); X509Certificate certificate = instanceIdentity.certificate(); - writeCredentialsToDisk(newKeyPair.getPrivate(), certificate, document); - return new AthenzCredentials(certificate, newKeyPair, document); + writeCredentialsToDisk(newKeyPair.getPrivate(), certificate, signedDocument); + return new AthenzCredentials(certificate, newKeyPair, signedDocument); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java index 5b884e3dfb3..48fc021dced 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java @@ -56,16 +56,16 @@ public class DefaultIdentityDocumentClient implements IdentityDocumentClient { } @Override - public SignedIdentityDocument getNodeIdentityDocument(String host) { - return getIdentityDocument(host, "node"); + public SignedIdentityDocument getNodeIdentityDocument(String host, int documentVersion) { + return getIdentityDocument(host, "node", documentVersion); } @Override - public SignedIdentityDocument getTenantIdentityDocument(String host) { - return getIdentityDocument(host, "tenant"); + public SignedIdentityDocument getTenantIdentityDocument(String host, int documentVersion) { + return getIdentityDocument(host, "tenant", documentVersion); } - private SignedIdentityDocument getIdentityDocument(String host, String type) { + private SignedIdentityDocument getIdentityDocument(String host, String type, int documentVersion) { try (CloseableHttpClient client = createHttpClient(sslContextSupplier.get(), hostnameVerifier)) { URI uri = configserverUri @@ -76,6 +76,7 @@ public class DefaultIdentityDocumentClient implements IdentityDocumentClient { .setUri(uri) .addHeader("Connection", "close") .addHeader("Accept", "application/json") + .addParameter("documentVersion", Integer.toString(documentVersion)) .build(); try (CloseableHttpResponse response = client.execute(request)) { String responseContent = EntityUtils.toString(response.getEntity()); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java index 019f73fc6bf..11b30585933 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java @@ -4,7 +4,10 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.security.SignatureUtils; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.identityprovider.api.DefaultSignedIdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; +import com.yahoo.vespa.athenz.identityprovider.api.LegacySignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; @@ -19,7 +22,7 @@ import java.util.Base64; import java.util.Set; import java.util.TreeSet; -import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION; +import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION; import static java.nio.charset.StandardCharsets.UTF_8; /** @@ -29,8 +32,25 @@ import static java.nio.charset.StandardCharsets.UTF_8; */ public class IdentityDocumentSigner { + public String generateSignature(String identityDocumentData, PrivateKey privateKey) { + try { + Signature signer = SignatureUtils.createSigner(privateKey); + signer.initSign(privateKey); + signer.update(identityDocumentData.getBytes(UTF_8)); + byte[] signature = signer.sign(); + return Base64.getEncoder().encodeToString(signature); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + } + + public String generateLegacySignature(IdentityDocument doc, PrivateKey privateKey) { + return generateSignature(doc.providerUniqueId(), doc.providerService(), doc.configServerHostname(), + doc.instanceHostname(), doc.createdAt(), doc.ipAddresses(), doc.identityType(), privateKey, doc.serviceIdentity()); + } + // Cluster type is ignored due to old Vespa versions not forwarding unknown fields in signed identity document - public String generateSignature(VespaUniqueInstanceId providerUniqueId, + private String generateSignature(VespaUniqueInstanceId providerUniqueId, AthenzService providerService, String configServerHostname, String instanceHostname, @@ -54,14 +74,32 @@ public class IdentityDocumentSigner { } public boolean hasValidSignature(SignedIdentityDocument doc, PublicKey publicKey) { + if (doc instanceof LegacySignedIdentityDocument signedDoc) { + return validateLegacySignature(signedDoc, publicKey); + } else if (doc instanceof DefaultSignedIdentityDocument signedDoc) { + try { + Signature signer = SignatureUtils.createVerifier(publicKey); + signer.initVerify(publicKey); + signer.update(signedDoc.data().getBytes(UTF_8)); + return signer.verify(Base64.getDecoder().decode(doc.signature())); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + } else { + throw new IllegalArgumentException("Unknown identity document type: " + doc.getClass().getName()); + } + } + + private boolean validateLegacySignature(SignedIdentityDocument doc, PublicKey publicKey) { try { + IdentityDocument iddoc = doc.identityDocument(); Signature signer = SignatureUtils.createVerifier(publicKey); signer.initVerify(publicKey); writeToSigner( - signer, doc.providerUniqueId(), doc.providerService(), doc.configServerHostname(), - doc.instanceHostname(), doc.createdAt(), doc.ipAddresses(), doc.identityType()); - if (doc.documentVersion() >= DEFAULT_DOCUMENT_VERSION) { - writeToSigner(signer, doc.serviceIdentity()); + signer, iddoc.providerUniqueId(), iddoc.providerService(), iddoc.configServerHostname(), + iddoc.instanceHostname(), iddoc.createdAt(), iddoc.ipAddresses(), iddoc.identityType()); + if (doc.documentVersion() >= LEGACY_DEFAULT_DOCUMENT_VERSION) { + writeToSigner(signer, iddoc.serviceIdentity()); } return signer.verify(Base64.getDecoder().decode(doc.signature())); } catch (GeneralSecurityException e) { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java index e67af028402..d699564a4ee 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java @@ -298,7 +298,7 @@ public final class LegacyAthenzIdentityProviderImpl extends AbstractComponent im } private X509Certificate requestRoleCertificate(AthenzRole role) { - var doc = credentials.getIdentityDocument(); + var doc = credentials.getIdentityDocument().identityDocument(); Pkcs10Csr csr = csrGenerator.generateRoleCsr( identity, role, doc.providerUniqueId(), doc.clusterType(), credentials.getKeyPair()); try (ZtsClient client = createZtsClient()) { diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java index 2a68f6fd231..513fb4cdbd3 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java @@ -5,8 +5,11 @@ package com.yahoo.vespa.athenz.identityprovider.api; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -15,7 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class EntityBindingsMapperTest { @Test - public void persists_unknown_json_members() throws IOException { + public void legacy_persists_unknown_json_members() throws IOException { var originalJson = """ { @@ -36,7 +39,8 @@ class EntityBindingsMapperTest { } """; var entity = EntityBindingsMapper.fromString(originalJson); - assertEquals(2, entity.unknownAttributes().size(), entity.unknownAttributes().toString()); + assertInstanceOf(LegacySignedIdentityDocument.class, entity); + assertEquals(2, entity.identityDocument().unknownAttributes().size(), entity.identityDocument().unknownAttributes().toString()); var json = EntityBindingsMapper.toAttestationData(entity); var expectedMemberInJson = "member-in-unknown-object"; @@ -45,4 +49,39 @@ class EntityBindingsMapperTest { assertEquals(EntityBindingsMapper.mapper.readTree(originalJson), EntityBindingsMapper.mapper.readTree(json)); } + @Test + public void reads_unknown_json_members() throws IOException { + var iddoc = """ + { + "provider-unique-id": "0.cluster.instance.app.tenant.us-west-1.test.node", + "provider-service": "domain.service", + "configserver-hostname": "cfg", + "instance-hostname": "host", + "created-at": 12345.0, + "ip-addresses": [], + "identity-type": "node", + "cluster-type": "admin", + "zts-url": "https://zts.url/", + "unknown-string": "string-value", + "unknown-object": { "member-in-unknown-object": 123 } + } + """; + var originalJson = + """ + { + "signature": "sig", + "signing-key-version": 0, + "document-version": 4, + "data": "%s" + } + """.formatted(Base64.getEncoder().encodeToString(iddoc.getBytes(StandardCharsets.UTF_8))); + var entity = EntityBindingsMapper.fromString(originalJson); + assertEquals(2, entity.identityDocument().unknownAttributes().size(), entity.identityDocument().unknownAttributes().toString()); + var json = EntityBindingsMapper.toAttestationData(entity); + + // For the new iddoc format the identity document should be unchanged during serialization/deserialization, + // i.e the signed identity document should be unchanged + assertEquals(EntityBindingsMapper.mapper.readTree(originalJson), EntityBindingsMapper.mapper.readTree(json)); + } + }
\ No newline at end of file diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java index ff85cb79f02..acb0905700f 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java @@ -6,10 +6,13 @@ import com.yahoo.security.KeyUtils; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; +import com.yahoo.vespa.athenz.identityprovider.api.DefaultSignedIdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; +import com.yahoo.vespa.athenz.identityprovider.api.LegacySignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.utils.AthenzIdentities; import org.junit.jupiter.api.Test; import java.security.KeyPair; @@ -18,6 +21,7 @@ import java.util.Arrays; import java.util.HashSet; import static com.yahoo.vespa.athenz.identityprovider.api.IdentityType.TENANT; +import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION; import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -42,32 +46,53 @@ public class IdentityDocumentSignerTest { private static final AthenzIdentity serviceIdentity = new AthenzService("vespa", "node"); @Test - void generates_and_validates_signature() { + void legacy_generates_and_validates_signature() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); + IdentityDocument identityDocument = new IdentityDocument( + id, providerService, configserverHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); String signature = - signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, - ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); + signer.generateLegacySignature(identityDocument, keyPair.getPrivate()); - SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( - signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); + SignedIdentityDocument signedIdentityDocument = new LegacySignedIdentityDocument( + signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, identityDocument); assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); } @Test - void ignores_cluster_type_and_zts_url() { + void generates_and_validates_signature() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); + IdentityDocument identityDocument = new IdentityDocument( + id, providerService, configserverHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); + String data = EntityBindingsMapper.toIdentityDocmentData(identityDocument); String signature = - signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, - ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); + signer.generateSignature(data, keyPair.getPrivate()); - var docWithoutIgnoredFields = new SignedIdentityDocument( - signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, null, null, serviceIdentity); - var docWithIgnoredFields = new SignedIdentityDocument( - signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, + SignedIdentityDocument signedIdentityDocument = new DefaultSignedIdentityDocument( + signature, KEY_VERSION, DEFAULT_DOCUMENT_VERSION, data); + + assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); + } + + @Test + void legacy_ignores_cluster_type_and_zts_url() { + IdentityDocumentSigner signer = new IdentityDocumentSigner(); + IdentityDocument identityDocument = new IdentityDocument( + id, providerService, configserverHostname, instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); + IdentityDocument withoutIgnoredFields = new IdentityDocument( + id, providerService, configserverHostname, + instanceHostname, createdAt, ipAddresses, identityType, null, null, serviceIdentity); + + String signature = + signer.generateLegacySignature(identityDocument, keyPair.getPrivate()); + + var docWithoutIgnoredFields = new LegacySignedIdentityDocument( + signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, withoutIgnoredFields); + var docWithIgnoredFields = new LegacySignedIdentityDocument( + signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, identityDocument); assertTrue(signer.hasValidSignature(docWithoutIgnoredFields, keyPair.getPublic())); assertEquals(docWithIgnoredFields.signature(), docWithoutIgnoredFields.signature()); @@ -76,16 +101,15 @@ public class IdentityDocumentSignerTest { @Test void validates_signature_for_new_and_old_versions() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); + IdentityDocument identityDocument = new IdentityDocument( + id, providerService, configserverHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); String signature = - signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, - ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); + signer.generateLegacySignature(identityDocument, keyPair.getPrivate()); - SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( - signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); + SignedIdentityDocument signedIdentityDocument = new LegacySignedIdentityDocument( + signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, identityDocument); assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); - } - }
\ No newline at end of file diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 0d007097fa2..1c2db273653 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -154,20 +154,20 @@ org.codehaus.plexus:plexus-sec-dispatcher:2.0 org.codehaus.plexus:plexus-utils:3.3.1 org.eclipse.collections:eclipse-collections:11.0.0 org.eclipse.collections:eclipse-collections-api:11.0.0 -org.eclipse.jetty:jetty-alpn-client:11.0.14 -org.eclipse.jetty:jetty-alpn-java-server:11.0.14 -org.eclipse.jetty:jetty-alpn-server:11.0.14 -org.eclipse.jetty:jetty-client:11.0.14 -org.eclipse.jetty:jetty-http:11.0.14 -org.eclipse.jetty:jetty-io:11.0.14 -org.eclipse.jetty:jetty-jmx:11.0.14 -org.eclipse.jetty:jetty-security:11.0.14 -org.eclipse.jetty:jetty-server:11.0.14 -org.eclipse.jetty:jetty-servlet:11.0.14 -org.eclipse.jetty:jetty-util:11.0.14 -org.eclipse.jetty.http2:http2-common:11.0.14 -org.eclipse.jetty.http2:http2-hpack:11.0.14 -org.eclipse.jetty.http2:http2-server:11.0.14 +org.eclipse.jetty:jetty-alpn-client:11.0.15 +org.eclipse.jetty:jetty-alpn-java-server:11.0.15 +org.eclipse.jetty:jetty-alpn-server:11.0.15 +org.eclipse.jetty:jetty-client:11.0.15 +org.eclipse.jetty:jetty-http:11.0.15 +org.eclipse.jetty:jetty-io:11.0.15 +org.eclipse.jetty:jetty-jmx:11.0.15 +org.eclipse.jetty:jetty-security:11.0.15 +org.eclipse.jetty:jetty-server:11.0.15 +org.eclipse.jetty:jetty-servlet:11.0.15 +org.eclipse.jetty:jetty-util:11.0.15 +org.eclipse.jetty.http2:http2-common:11.0.15 +org.eclipse.jetty.http2:http2-hpack:11.0.15 +org.eclipse.jetty.http2:http2-server:11.0.15 org.eclipse.jetty.toolchain:jetty-jakarta-servlet-api:5.0.2 org.eclipse.sisu:org.eclipse.sisu.inject:0.3.5 org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.5 diff --git a/vespa-documentgen-plugin/etc/music/music.sd b/vespa-documentgen-plugin/etc/music/music.sd index 205614db67d..8c2d324697c 100644 --- a/vespa-documentgen-plugin/etc/music/music.sd +++ b/vespa-documentgen-plugin/etc/music/music.sd @@ -32,6 +32,14 @@ search music { indexing: summary | index } + field tags type weightedset<string> { + indexing: attribute | summary + attribute: fast-search + weightedset { + create-if-nonexistent + } + } + } rank-profile default inherits default { @@ -48,6 +56,11 @@ search music { } + document-summary tags { + summary tags type weightedset<string> { + full + } + } } diff --git a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java index 2a7eb261a0b..f55b226b11b 100644 --- a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java +++ b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java @@ -21,9 +21,11 @@ public class DocumentGenTest { DocumentGenMojo mojo = new DocumentGenMojo(); mojo.execute(new File("etc/music/"), new File("target/generated-test-sources/vespa-documentgen-plugin/"), "com.yahoo.vespa.document"); Map<String, Schema> searches = mojo.getSearches(); - assertEquals(searches.size(),1); + assertEquals(searches.size(), 1); assertEquals(searches.get("music").getDocument("music").getField("title").getDataType(), DataType.STRING); assertEquals(searches.get("music").getDocument("music").getField("eitheror").getDataType(), DataType.BOOL); + assertEquals(searches.get("music").getDocument("music").getField("tags").getDataType(), + searches.get("music").getSummaries().get("tags").getSummaryField("tags").getDataType()); } @Test diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 5126f7e0f43..45c93ec0755 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -132,6 +132,7 @@ import static java.util.stream.Collectors.toUnmodifiableMap; public class DocumentV1ApiHandler extends AbstractRequestHandler { private static final Duration defaultTimeout = Duration.ofSeconds(180); // Match document API default timeout. + private static final Duration handlerTimeout = Duration.ofMillis(100); // Extra time to allow for handler, JDisc and jetty to complete. private static final Logger log = Logger.getLogger(DocumentV1ApiHandler.class.getName()); private static final Parser<Integer> integerParser = Integer::parseInt; @@ -175,7 +176,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private static final String TO_TIMESTAMP = "toTimestamp"; private final Clock clock; - private final Duration handlerTimeout; + private final Duration visitTimeout; private final Metric metric; private final DocumentApiMetrics metrics; private final DocumentOperationParser parser; @@ -204,11 +205,11 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { documentManagerConfig, executorConfig, clusterListConfig, bucketSpacesConfig); } - DocumentV1ApiHandler(Clock clock, Duration handlerTimeout, Metric metric, MetricReceiver metricReceiver, DocumentAccess access, + DocumentV1ApiHandler(Clock clock, Duration visitTimeout, Metric metric, MetricReceiver metricReceiver, DocumentAccess access, DocumentmanagerConfig documentmanagerConfig, DocumentOperationExecutorConfig executorConfig, ClusterListConfig clusterListConfig, AllClustersBucketSpacesConfig bucketSpacesConfig) { this.clock = clock; - this.handlerTimeout = handlerTimeout; + this.visitTimeout = visitTimeout; this.parser = new DocumentOperationParser(documentmanagerConfig); this.metric = metric; this.metrics = new DocumentApiMetrics(metricReceiver, "documentV1"); @@ -237,9 +238,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { HttpRequest request = (HttpRequest) rawRequest; try { // Set a higher HTTP layer timeout than the document API timeout, to prefer triggering the latter. - request.setTimeout( getProperty(request, TIMEOUT, timeoutMillisParser).orElse(defaultTimeout.toMillis()) - + handlerTimeout.toMillis(), - MILLISECONDS); + request.setTimeout(doomMillis(request) - clock.millis(), MILLISECONDS); Path requestPath = Path.withoutValidation(request.getUri()); // No segment validation here, as document IDs can be anything. for (String path : handlers.keySet()) { @@ -267,7 +266,8 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { @Override public void handleTimeout(Request request, ResponseHandler responseHandler) { - timeout((HttpRequest) request, "Timeout after " + (request.getTimeout(MILLISECONDS) - handlerTimeout.toMillis()) + "ms", responseHandler); + HttpRequest httpRequest = (HttpRequest) request; + timeout(httpRequest, "Timeout after " + (getProperty(httpRequest, TIMEOUT, timeoutMillisParser).orElse(defaultTimeout.toMillis())) + "ms", responseHandler); } @Override @@ -523,9 +523,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private DocumentOperationParameters parametersFromRequest(HttpRequest request, String... names) { DocumentOperationParameters parameters = getProperty(request, TRACELEVEL, integerParser).map(parameters()::withTraceLevel) .orElse(parameters()); - parameters = getProperty(request, TIMEOUT, timeoutMillisParser).map(clock.instant()::plusMillis) - .map(parameters::withDeadline) - .orElse(parameters); + parameters = parameters.withDeadline(Instant.ofEpochMilli(doomMillis(request)).minus(handlerTimeout)); for (String name : names) parameters = switch (name) { case CLUSTER -> @@ -1168,16 +1166,15 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { parameters.setFieldSet(getProperty(request, FIELD_SET).orElse(path.documentType().map(type -> type + ":[document]").orElse(DocumentOnly.NAME))); parameters.setMaxTotalHits(wantedDocumentCount); parameters.visitInconsistentBuckets(true); - long timeoutMs = Math.max(1, request.getTimeout(MILLISECONDS) - handlerTimeout.toMillis()); if (streamed) { StaticThrottlePolicy throttlePolicy = new DynamicThrottlePolicy().setMinWindowSize(1).setWindowSizeIncrement(1); concurrency.ifPresent(throttlePolicy::setMaxPendingCount); parameters.setThrottlePolicy(throttlePolicy); - parameters.setTimeoutMs(timeoutMs); // Ensure visitor eventually completes. + parameters.setTimeoutMs(visitTimeout(request)); // Ensure visitor eventually completes. } else { parameters.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(Math.min(100, concurrency.orElse(1)))); - parameters.setSessionTimeoutMs(timeoutMs); + parameters.setSessionTimeoutMs(visitTimeout(request)); } return parameters; } @@ -1188,10 +1185,16 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { VisitorParameters parameters = parseCommonParameters(request, path, Optional.of(requireProperty(request, CLUSTER))); parameters.setThrottlePolicy(new DynamicThrottlePolicy().setMinWindowSize(1).setWindowSizeIncrement(1)); long timeChunk = getProperty(request, TIME_CHUNK, timeoutMillisParser).orElse(60_000L); - parameters.setSessionTimeoutMs(Math.max(1, Math.min(timeChunk, request.getTimeout(MILLISECONDS) - handlerTimeout.toMillis()))); + parameters.setSessionTimeoutMs(Math.min(timeChunk, visitTimeout(request))); return parameters; } + private long visitTimeout(HttpRequest request) { + return Math.max(1, + Math.max(doomMillis(request) - clock.millis() - visitTimeout.toMillis(), + 9 * (doomMillis(request) - clock.millis()) / 10 - handlerTimeout.toMillis())); + } + private VisitorParameters parseCommonParameters(HttpRequest request, DocumentPath path, Optional<String> cluster) { VisitorParameters parameters = new VisitorParameters(Stream.of(getProperty(request, SELECTION), path.documentType(), @@ -1345,7 +1348,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { AtomicReference<String> error = new AtomicReference<>(); // Set if error occurs during processing of visited documents. callback.onStart(response, fullyApplied); VisitorControlHandler controller = new VisitorControlHandler() { - final ScheduledFuture<?> abort = streaming ? visitDispatcher.schedule(this::abort, request.getTimeout(MILLISECONDS), MILLISECONDS) : null; + final ScheduledFuture<?> abort = streaming ? visitDispatcher.schedule(this::abort, visitTimeout(request), MILLISECONDS) : null; final AtomicReference<VisitorSession> session = new AtomicReference<>(); @Override public void setSession(VisitorControlSession session) { // Workaround for broken session API ಠ_ಠ super.setSession(session); @@ -1426,6 +1429,12 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { // ------------------------------------------------ Helpers ------------------------------------------------ + private static long doomMillis(HttpRequest request) { + long createdAtMillis = request.creationTime(MILLISECONDS); + long requestTimeoutMillis = getProperty(request, TIMEOUT, timeoutMillisParser).orElse(defaultTimeout.toMillis()); + return createdAtMillis + requestTimeoutMillis; + } + private static String requireProperty(HttpRequest request, String name) { return getProperty(request, name) .orElseThrow(() -> new IllegalArgumentException("Must specify '" + name + "' at '" + request.getUri().getRawPath() + "'")); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 7696fd2196c..1d81c45daf1 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -89,6 +89,7 @@ import static com.yahoo.jdisc.http.HttpRequest.Method.POST; import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -221,7 +222,7 @@ public class DocumentV1ApiTest { assertEquals(100, ((StaticThrottlePolicy) parameters.getThrottlePolicy()).getMaxPendingCount()); assertEquals("[id]", parameters.getFieldSet()); assertEquals("(all the things)", parameters.getDocumentSelection()); - assertEquals(6000, parameters.getSessionTimeoutMs()); + assertTrue(6000 <= parameters.getSessionTimeoutMs()); // Static clock in handler < connected time for request, test artefact. assertEquals(9, parameters.getTraceLevel()); assertEquals(1_000_000, parameters.getFromTimestamp()); assertEquals(2_000_000, parameters.getToTimestamp()); @@ -283,7 +284,7 @@ public class DocumentV1ApiTest { assertEquals(1, ((StaticThrottlePolicy) parameters.getThrottlePolicy()).getMaxPendingCount()); assertEquals("[id]", parameters.getFieldSet()); assertEquals("(all the things)", parameters.getDocumentSelection()); - assertEquals(6000, parameters.getTimeoutMs()); + assertTrue(6000 <= parameters.getTimeoutMs()); // Static clock in handler < connected time for request, test artefact. assertEquals(4, parameters.getSlices()); assertEquals(1, parameters.getSliceId()); assertEquals(0, parameters.getFromTimestamp()); // not set; 0 is default @@ -812,7 +813,7 @@ public class DocumentV1ApiTest { // TIMEOUT is a 504 access.session.expect((id, parameters) -> { - assertEquals(clock.instant().plusSeconds(1000), parameters.deadline().get()); + assertFalse(clock.instant().plusSeconds(1000).isAfter(parameters.deadline().get())); // Static clock in handler vs real clock in Request. parameters.responseHandler().get().handleResponse(new Response(0, "timeout", Response.Outcome.TIMEOUT)); return new Result(); }); diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryView.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryView.java index 0e111d42061..1b4468d18bb 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BinaryView.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryView.java @@ -14,15 +14,24 @@ public final class BinaryView implements Inspector { private final byte[] data; private final SymbolTable names; - private final DecodeIndex index; + private final long[] index; private final int self; - private BinaryView(byte[] data, SymbolTable names, DecodeIndex index, int self) { + private BinaryView(byte[] data, SymbolTable names, long[] index, int self) { this.data = data; this.names = names; this.index = index; this.self = self; } + private int byte_offset(int idx) { + return (int)(index[idx] >> 33) & 0x7fff_ffff; + } + private int first_child(int idx) { + return (int)(index[idx] >> 2) & 0x7fff_ffff; + } + private int ext_bits(int idx) { + return (int)index[idx] & 0x3; + } private int peek_cmpr_int(int idx) { long next = data[idx++]; long value = (next & 0x7f); @@ -92,8 +101,8 @@ public final class BinaryView implements Inspector { } private Inspector find_field(int pos, int len, int sym) { for (int i = 0; i < len; ++i) { - int idx = index.getByteOffset(pos + i); - if (peek_cmpr_int(idx - (index.getExtBits(pos + i) + 1)) == sym) { + int idx = byte_offset(pos + i); + if (peek_cmpr_int(idx - (ext_bits(pos + i) + 1)) == sym) { return new BinaryView(data, names, index, pos + i); } } @@ -102,110 +111,110 @@ public final class BinaryView implements Inspector { @Override public boolean valid() { return true; } @Override public void ifValid(Consumer<Inspector> consumer) { consumer.accept(this); } - @Override public Type type() { return decode_type(data[index.getByteOffset(self)]); } + @Override public Type type() { return decode_type(data[byte_offset(self)]); } @Override public int children() { return switch (type()) { - case OBJECT, ARRAY -> extract_children(index.getByteOffset(self)); + case OBJECT, ARRAY -> extract_children(byte_offset(self)); default -> 0; }; } @Override public int entries() { return switch (type()) { - case ARRAY -> extract_children(index.getByteOffset(self)); + case ARRAY -> extract_children(byte_offset(self)); default -> 0; }; } @Override public int fields() { return switch (type()) { - case OBJECT -> extract_children(index.getByteOffset(self)); + case OBJECT -> extract_children(byte_offset(self)); default -> 0; }; } @Override public boolean asBool() { return switch (type()) { - case BOOL -> (decode_meta(data[index.getByteOffset(self)]) != 0); + case BOOL -> (decode_meta(data[byte_offset(self)]) != 0); default -> false; }; } @Override public long asLong() { return switch (type()) { - case LONG -> extract_long(index.getByteOffset(self)); - case DOUBLE -> (long)extract_double(index.getByteOffset(self)); + case LONG -> extract_long(byte_offset(self)); + case DOUBLE -> (long)extract_double(byte_offset(self)); default -> 0; }; } @Override public double asDouble() { return switch (type()) { - case LONG -> extract_long(index.getByteOffset(self)); - case DOUBLE -> extract_double(index.getByteOffset(self)); + case LONG -> extract_long(byte_offset(self)); + case DOUBLE -> extract_double(byte_offset(self)); default -> 0.0; }; } @Override public String asString() { return switch (type()) { - case STRING -> extract_string(index.getByteOffset(self)); + case STRING -> extract_string(byte_offset(self)); default -> Value.emptyString; }; } @Override public byte[] asUtf8() { return switch (type()) { - case STRING -> extract_bytes(index.getByteOffset(self)); + case STRING -> extract_bytes(byte_offset(self)); default -> Value.emptyData; }; } @Override public byte[] asData() { return switch (type()) { - case DATA -> extract_bytes(index.getByteOffset(self)); + case DATA -> extract_bytes(byte_offset(self)); default -> Value.emptyData; }; } @Override public void accept(Visitor v) { switch (type()) { case NIX: v.visitNix(); break; - case BOOL: v.visitBool(decode_meta(data[index.getByteOffset(self)]) != 0); break; - case LONG: v.visitLong(extract_long(index.getByteOffset(self))); break; - case DOUBLE: v.visitDouble(extract_double(index.getByteOffset(self))); break; - case STRING: v.visitString(extract_bytes(index.getByteOffset(self))); break; - case DATA: v.visitData(extract_bytes(index.getByteOffset(self))); break; + case BOOL: v.visitBool(decode_meta(data[byte_offset(self)]) != 0); break; + case LONG: v.visitLong(extract_long(byte_offset(self))); break; + case DOUBLE: v.visitDouble(extract_double(byte_offset(self))); break; + case STRING: v.visitString(extract_bytes(byte_offset(self))); break; + case DATA: v.visitData(extract_bytes(byte_offset(self))); break; case ARRAY: v.visitArray(this); break; case OBJECT: v.visitObject(this); break; default: throw new RuntimeException("should not be reached"); } } @Override public void traverse(ArrayTraverser at) { - int pos = index.getFirstChild(self); + int pos = first_child(self); int len = entries(); for (int i = 0; i < len; ++i) { at.entry(i, new BinaryView(data, names, index, pos + i)); } } @Override public void traverse(ObjectSymbolTraverser ot) { - int pos = index.getFirstChild(self); + int pos = first_child(self); int len = fields(); for (int i = 0; i < len; ++i) { - int sym = peek_cmpr_int(index.getByteOffset(pos + i) - (index.getExtBits(pos + i) + 1)); + int sym = peek_cmpr_int(byte_offset(pos + i) - (ext_bits(pos + i) + 1)); ot.field(sym, new BinaryView(data, names, index, pos + i)); } } @Override public void traverse(ObjectTraverser ot) { - int pos = index.getFirstChild(self); + int pos = first_child(self); int len = fields(); for (int i = 0; i < len; ++i) { - int sym = peek_cmpr_int(index.getByteOffset(pos + i) - (index.getExtBits(pos + i) + 1)); + int sym = peek_cmpr_int(byte_offset(pos + i) - (ext_bits(pos + i) + 1)); ot.field(names.inspect(sym), new BinaryView(data, names, index, pos + i)); } } @Override public Inspector entry(int idx) { int limit = entries(); if (idx >= 0 && idx < limit) { - return new BinaryView(data, names, index, index.getFirstChild(self) + idx); + return new BinaryView(data, names, index, first_child(self) + idx); } return NixValue.invalid(); } @Override public Inspector field(int sym) { int limit = fields(); if (limit > 0 && sym != SymbolTable.INVALID) { - return find_field(index.getFirstChild(self), limit, sym); + return find_field(first_child(self), limit, sym); } return NixValue.invalid(); } @@ -214,7 +223,7 @@ public final class BinaryView implements Inspector { if (limit > 0) { int sym = names.lookup(name); if (sym != SymbolTable.INVALID) { - return find_field(index.getFirstChild(self), limit, sym); + return find_field(first_child(self), limit, sym); } } return NixValue.invalid(); @@ -243,11 +252,11 @@ public final class BinaryView implements Inspector { break; } case ARRAY: { int size = input.read_size(meta); - if (size > input.getBacking().length - index.size()) { + int firstChild = index.tryReserveChildren(size, index.used() + 1, input.getPosition()); + if (firstChild < 0) { input.fail("decode index too big"); return; } - int firstChild = index.reserve(size); index.set(self, pos, firstChild, extBits); for (int i = 0; i < size; ++i) { buildIndex(input, index, firstChild + i, 0); @@ -255,11 +264,11 @@ public final class BinaryView implements Inspector { break; } case OBJECT: { int size = input.read_size(meta); - if (size > input.getBacking().length - index.size()) { + int firstChild = index.tryReserveChildren(size, index.used() + 1, input.getPosition()); + if (firstChild < 0) { input.fail("decode index too big"); return; } - int firstChild = index.reserve(size); index.set(self, pos, firstChild, extBits); for (int i = 0; i < size; ++i) { int childExtBits = input.skip_cmpr_int(); @@ -274,19 +283,16 @@ public final class BinaryView implements Inspector { } } - static Inspector inspectImpl(BufferedInput input) { + public static Inspector inspect(byte[] data) { + var input = new BufferedInput(data); var names = new SymbolTable(); - var index = new DecodeIndex(); BinaryDecoder.decodeSymbolTable(input, names); - buildIndex(input, index, index.reserve(1), 0); + var index = new DecodeIndex(input.getBacking().length, input.getPosition()); + buildIndex(input, index, 0, 0); if (input.failed()) { - return NixValue.invalid(); + throw new IllegalArgumentException("bad input: " + input.getErrorMessage()); } - return new BinaryView(input.getBacking(), names, index, 0); - } - - public static Inspector inspect(byte[] data) { - return inspectImpl(new BufferedInput(data)); + return new BinaryView(input.getBacking(), names, index.getBacking(), 0); } static int peek_cmpr_int_for_testing(byte[] data, int idx) { @@ -304,4 +310,13 @@ public final class BinaryView implements Inspector { static double extract_double_for_testing(byte[] data, int idx) { return new BinaryView(data, null, null, -1).extract_double(idx); } + static int byte_offset_for_testing(DecodeIndex index, int idx) { + return new BinaryView(null, null, index.getBacking(), -1).byte_offset(idx); + } + static int first_child_for_testing(DecodeIndex index, int idx) { + return new BinaryView(null, null, index.getBacking(), -1).first_child(idx); + } + static int ext_bits_for_testing(DecodeIndex index, int idx) { + return new BinaryView(null, null, index.getBacking(), -1).ext_bits(idx); + } } diff --git a/vespajlib/src/main/java/com/yahoo/slime/DecodeIndex.java b/vespajlib/src/main/java/com/yahoo/slime/DecodeIndex.java index 17c7a86730e..645eac3b4d9 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/DecodeIndex.java +++ b/vespajlib/src/main/java/com/yahoo/slime/DecodeIndex.java @@ -6,23 +6,47 @@ package com.yahoo.slime; * encoded in binary format. **/ final class DecodeIndex { - static final int initial_capacity = 16; - private long[] data = new long[initial_capacity]; - private int reserved = 0; - - private int adjustSize(int minSize) { - int capacity = initial_capacity; - while (capacity < minSize) { - capacity = capacity << 1; + private long[] data; + private int reserved; + private int used = 0; + private final int totalSize; + private final int rootOffset; + + private int binarySize() { return totalSize - rootOffset; } + + private int adjustSize(int minSize, int maxSize, int cnt, int byteOffset) { + double density = (double)cnt / (double)(byteOffset - rootOffset); + double estSize = 1.1 * density * binarySize(); + double expSize = 1.25 * data.length; + double wantedSize = (estSize > expSize) ? estSize : expSize; + if (wantedSize < minSize) { + return minSize; + } + if (wantedSize > maxSize) { + return maxSize; } - return capacity; + return (int)wantedSize; } - int reserve(int n) { + DecodeIndex(int totalSize, int rootOffset) { + this.totalSize = totalSize; + this.rootOffset = rootOffset; + int initialCapacity = Math.max(16, binarySize() / 24); + data = new long[initialCapacity]; + reserved = 1; + } + + long[] getBacking() { return data; } + + int tryReserveChildren(int n, int cnt, int byteOffset) { int offset = reserved; - if (reserved + n > data.length) { + if (n > data.length - reserved) { + final int maxSize = (totalSize - byteOffset) + cnt; + if (n > maxSize - reserved) { + return -1; // error; too much space requested + } long[] old = data; - data = new long[adjustSize(reserved + n)]; + data = new long[adjustSize(reserved + n, maxSize, cnt, byteOffset)]; System.arraycopy(old, 0, data, 0, reserved); } reserved += n; @@ -30,22 +54,13 @@ final class DecodeIndex { } int size() { return reserved; } + int used() { return used; } + int capacity() { return data.length; } void set(int idx, int byteOffset, int firstChild, int extBits) { data[idx] = (long)(byteOffset & 0x7fff_ffff) << 33 | (long)(firstChild & 0x7fff_ffff) << 2 | extBits & 0x3; - } - - int getByteOffset(int idx) { - return (int)(data[idx] >> 33) & 0x7fff_ffff; - } - - int getFirstChild(int idx) { - return (int)(data[idx] >> 2) & 0x7fff_ffff; - } - - int getExtBits(int idx) { - return (int)data[idx] & 0x3; + ++used; } } diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java index 920a25b96c9..99c63c91afc 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java +++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java @@ -100,7 +100,15 @@ public class BinaryViewTest { assertEquals(300, arr.entries()); return arr; } - static final int numShapes = numLeafs + 6; + static Cursor insert10SimpleHits(Inserter dst) { + var arr = dst.insertARRAY(); + for (int i = 0; i < 10; ++i) { + var obj = arr.addObject(); + obj.setLong("id", 123456); + } + return arr; + } + static final int numShapes = numLeafs + 7; static Cursor insertRoot(Slime dst, int shape) { var root = new SlimeInserter(dst); if (shape < numLeafs) { @@ -113,6 +121,7 @@ public class BinaryViewTest { case (numLeafs + 3) -> insertOuterArray(root); case (numLeafs + 4) -> insertManySymbols(root); case (numLeafs + 5) -> insertLargeArray(root); + case (numLeafs + 6) -> insert10SimpleHits(root); default -> NixValue.invalid(); }; } @@ -297,66 +306,49 @@ public class BinaryViewTest { } } + void assertFail(byte[] data, String reason) { + try { + var view = BinaryView.inspect(data); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("bad input: " + reason, e.getMessage()); + } + } + @Test public void testTrivialView() { byte[] data = {0, 0}; - var input = new BufferedInput(data); - var view = BinaryView.inspectImpl(input); + var view = BinaryView.inspect(data); assertTrue(view.valid()); assertEquals(Type.NIX, view.type()); - assertFalse(input.failed()); } @Test public void testUnderflow() { byte[] data = {}; - var input = new BufferedInput(data); - var view = BinaryView.inspectImpl(input); - assertFalse(view.valid()); - assertTrue(input.failed()); - assertEquals("underflow", input.getErrorMessage()); + assertFail(data, "underflow"); } @Test public void testMultiByteUnderflow() { byte[] data = { 0, encode_type_and_meta(Type.STRING.ID, 3), 65 }; - var input = new BufferedInput(data); - var view = BinaryView.inspectImpl(input); - assertFalse(view.valid()); - assertTrue(input.failed()); - assertEquals("underflow", input.getErrorMessage()); + assertFail(data, "underflow"); } @Test public void testCompressedIntOverflow() { byte[] data = { -1, -1, -1, -1, 8 }; - var input = new BufferedInput(data); - var view = BinaryView.inspectImpl(input); - assertFalse(view.valid()); - assertTrue(input.failed()); - assertEquals("compressed int overflow", input.getErrorMessage()); + assertFail(data, "compressed int overflow"); } @Test public void testExtBitsOverflow() { byte[] data = { 0, encode_type_and_meta(Type.OBJECT.ID, 2), -1, -1, -1, -1, 1 }; - var input = new BufferedInput(data); - var view = BinaryView.inspectImpl(input); - assertFalse(view.valid()); - assertTrue(input.failed()); - assertEquals("symbol id too big", input.getErrorMessage()); + assertFail(data, "symbol id too big"); } @Test public void testDecodeIndexOverflowArray() { - byte[] data = { 0, encode_type_and_meta(Type.ARRAY.ID, 4) }; - var input = new BufferedInput(data); - var view = BinaryView.inspectImpl(input); - assertFalse(view.valid()); - assertTrue(input.failed()); - assertEquals("decode index too big", input.getErrorMessage()); + byte[] data = { 0, encode_type_and_meta(Type.ARRAY.ID, 20) }; + assertFail(data, "decode index too big"); } @Test public void testDecodeIndexOverflowObject() { - byte[] data = { 0, encode_type_and_meta(Type.OBJECT.ID, 4) }; - var input = new BufferedInput(data); - var view = BinaryView.inspectImpl(input); - assertFalse(view.valid()); - assertTrue(input.failed()); - assertEquals("decode index too big", input.getErrorMessage()); + byte[] data = { 0, encode_type_and_meta(Type.OBJECT.ID, 20) }; + assertFail(data, "decode index too big"); } } diff --git a/vespajlib/src/test/java/com/yahoo/slime/DecodeIndexTest.java b/vespajlib/src/test/java/com/yahoo/slime/DecodeIndexTest.java index 223701fa2fd..c34a718d2bb 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/DecodeIndexTest.java +++ b/vespajlib/src/test/java/com/yahoo/slime/DecodeIndexTest.java @@ -3,83 +3,236 @@ package com.yahoo.slime; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static com.yahoo.slime.BinaryView.byte_offset_for_testing; +import static com.yahoo.slime.BinaryView.first_child_for_testing; +import static com.yahoo.slime.BinaryView.ext_bits_for_testing; public class DecodeIndexTest { + int checkCapacity(DecodeIndex index, int oldCapacity) { + int capacity = index.capacity(); + if (oldCapacity == -1) { + System.out.println("DecodeIndex initial capacity " + capacity); + } else if (capacity != oldCapacity) { + System.out.println("DecodeIndex capacity increased to " + capacity); + } + return capacity; + } + @Test public void testSimpleUsage() { - DecodeIndex index = new DecodeIndex(); - int val1 = index.reserve(1); - int val2 = index.reserve(3); - int val3 = index.reserve(2); - assertEquals(0, val1); + DecodeIndex index = new DecodeIndex(100, 10); + assertEquals(1, index.size()); + int capacity = checkCapacity(index, -1); + int root = 0; + capacity = checkCapacity(index, capacity); + int val2 = index.tryReserveChildren(3, 1, 15); + capacity = checkCapacity(index, capacity); + int val3 = index.tryReserveChildren(2, 2, 20); + capacity = checkCapacity(index, capacity); assertEquals(1, val2); assertEquals(4, val3); assertEquals(6, index.size()); - index.set(val1 + 0, 0, val2, 0); + index.set(root, 0, val2, 0); index.set(val2 + 0, 100, 0, 1); index.set(val2 + 1, 200, val3, 2); index.set(val2 + 2, 300, 0, 3); index.set(val3 + 0, 400, 0, 0); index.set(val3 + 1, 500, 0, 0); for (int i = 0; i < 6; i++) { - assertEquals(i * 100, index.getByteOffset(i)); + assertEquals(i * 100, byte_offset_for_testing(index, i)); if (i == 0) { - assertEquals(1, index.getFirstChild(i)); + assertEquals(1, first_child_for_testing(index, i)); } else if (i == 2) { - assertEquals(4, index.getFirstChild(i)); + assertEquals(4, first_child_for_testing(index, i)); } else { - assertEquals(0, index.getFirstChild(i)); + assertEquals(0, first_child_for_testing(index, i)); } if (i < 4) { - assertEquals(i, index.getExtBits(i)); + assertEquals(i, ext_bits_for_testing(index, i)); } else { - assertEquals(0, index.getExtBits(i)); + assertEquals(0, ext_bits_for_testing(index, i)); } } } @Test public void testManyValues() { - DecodeIndex index = new DecodeIndex(); int outer = 47; int inner = 73; - int expectOffset = 0; + int symSize = 128; + int bytesPerValue = 5; + DecodeIndex index = new DecodeIndex(symSize + inner * outer * bytesPerValue, symSize); + int capacity = checkCapacity(index, -1); + int indexOffset = 1; + int binaryOffset = symSize + bytesPerValue; + int expectOffset = 1; for (int i = 0; i < outer; i++) { - int offset = index.reserve(inner); + int offset = index.tryReserveChildren(inner, indexOffset, binaryOffset); + capacity = checkCapacity(index, capacity); assertEquals(expectOffset, offset); expectOffset += inner; for (int j = 0; j < inner; j++) { index.set(offset + j, (i * j), (i + j), (j & 3)); + ++indexOffset; + binaryOffset += bytesPerValue; } } - assertEquals(inner * outer, expectOffset); - assertEquals(inner * outer, index.size()); + assertEquals(1 + inner * outer, expectOffset); + assertEquals(1 + inner * outer, index.size()); for (int i = 0; i < outer; i++) { for (int j = 0; j < inner; j++) { - int offset = i * inner + j; - assertEquals(i * j, index.getByteOffset(offset)); - assertEquals(i + j, index.getFirstChild(offset)); - assertEquals(j & 3, index.getExtBits(offset)); + int offset = 1 + i * inner + j; + assertEquals(i * j, byte_offset_for_testing(index, offset)); + assertEquals(i + j, first_child_for_testing(index, offset)); + assertEquals(j & 3, ext_bits_for_testing(index, offset)); } } } @Test public void testOverflowNoBleed() { - DecodeIndex index = new DecodeIndex(); - index.reserve(3); + DecodeIndex index = new DecodeIndex(100, 10); + index.tryReserveChildren(2, 1, 20); + assertEquals(3, index.size()); index.set(0, 0xffff_ffff, 0, 0); index.set(1, 0, 0xffff_ffff, 0); index.set(2, 0, 0, 0xffff_ffff); - assertEquals(0x7fff_ffff, index.getByteOffset(0)); - assertEquals(0, index.getByteOffset(1)); - assertEquals(0, index.getByteOffset(2)); - assertEquals(0, index.getFirstChild(0)); - assertEquals(0x7fff_ffff, index.getFirstChild(1)); - assertEquals(0, index.getFirstChild(2)); - assertEquals(0, index.getExtBits(0)); - assertEquals(0, index.getExtBits(1)); - assertEquals(3, index.getExtBits(2)); + assertEquals(0x7fff_ffff, byte_offset_for_testing(index, 0)); + assertEquals(0, byte_offset_for_testing(index, 1)); + assertEquals(0, byte_offset_for_testing(index, 2)); + assertEquals(0, first_child_for_testing(index, 0)); + assertEquals(0x7fff_ffff, first_child_for_testing(index, 1)); + assertEquals(0, first_child_for_testing(index, 2)); + assertEquals(0, ext_bits_for_testing(index, 0)); + assertEquals(0, ext_bits_for_testing(index, 1)); + assertEquals(3, ext_bits_for_testing(index, 2)); + } + + @Test + public void testMinimalInitialCapacity() { + DecodeIndex index = new DecodeIndex(2, 1); + assertEquals(16, index.capacity()); + } + + @Test + public void testInitialCapacityEstimate() { + DecodeIndex index = new DecodeIndex((33 * 24) + 167, 167); + assertEquals(33, index.capacity()); + } + + void assertWithinRange(int low, int high, int actual) { + if (actual >= low && actual <= high) { + System.out.println("value " + actual + " in range [" + low + "," + high + "]"); + } else { + fail("value " + actual + " not in range [" + low + "," + high + "]"); + } + } + + void assertGreater(int limit, int actual) { + if (actual > limit) { + System.out.println("value " + actual + " is greater than " + limit); + } else { + fail("value " + actual + " is not greater than " + limit); + } + } + + void assertLess(int limit, int actual) { + if (actual < limit) { + System.out.println("value " + actual + " is less than " + limit); + } else { + fail("value " + actual + " is not less than " + limit); + } + } + + DecodeIndex prepareIndex(int symSize, int numValues) { + DecodeIndex index = new DecodeIndex((numValues * 24) + symSize, symSize); + assertEquals(1, index.tryReserveChildren(numValues - 1, 1, symSize + 24)); + assertEquals(numValues, index.size()); + assertEquals(numValues, index.capacity()); + return index; + } + + @Test + public void testDensityBasedCapacityEstimate() { + var index = prepareIndex(167, 33); + double exp = 1.25 * index.capacity(); + assertEquals(33, index.tryReserveChildren(10, 20, 167 + (20 * 4))); + int doneCnt = 20; + double bytesPerObject = 4.0; + int pendingData = (33 * 24) - (20 * 4); + double est = (doneCnt + pendingData / bytesPerObject); + int maxSize = doneCnt + pendingData; + assertGreater((int)(exp * 1.05), index.capacity()); + assertWithinRange((int)(1.05 * est), (int)(1.15 * est), index.capacity()); + assertLess(maxSize, index.capacity()); + } + + @Test + public void testExpCapacityGrowth() { + var index = prepareIndex(167, 33); + double exp = 1.25 * index.capacity(); + assertEquals(33, index.tryReserveChildren(1, 20, 167 + (20 * 32))); + int doneCnt = 20; + double bytesPerObject = 32.0; + int pendingData = (33 * 24) - (20 * 32); + double est = (doneCnt + pendingData / bytesPerObject); + int maxSize = doneCnt + pendingData; + assertWithinRange((int)(0.95 * exp), (int)(1.05 * exp), index.capacity()); + assertGreater((int)(est * 1.15), index.capacity()); + assertLess(maxSize, index.capacity()); + } + + @Test + public void testMinCapacityGrowth() { + var index = prepareIndex(167, 33); + double exp = 1.25 * index.capacity(); + assertEquals(33, index.tryReserveChildren(20, 20, 167 + (20 * 32))); + int doneCnt = 20; + double bytesPerObject = 32.0; + int pendingData = (33 * 24) - (20 * 32); + double est = (doneCnt + pendingData / bytesPerObject); + int maxSize = doneCnt + pendingData; + assertGreater((int)(exp * 1.05), index.capacity()); + assertGreater((int)(est * 1.15), index.capacity()); + assertEquals(33 + 20, index.capacity()); + assertLess(maxSize, index.capacity()); + } + + @Test + public void testMaxCapacityGrowth() { + var index = prepareIndex(167, 33); + double exp = 1.25 * index.capacity(); + assertEquals(33, index.tryReserveChildren(1, 32, 167 + (33 * 24) - 3)); + int minSize = 33 + 1; + int maxSize = 32 + 3; + assertLess((int)(exp * 0.95), index.capacity()); + assertGreater(minSize, index.capacity()); + assertEquals(maxSize, index.capacity()); + } + + @Test + public void testMinMaxCapacityGrowth() { + var index = prepareIndex(167, 33); + assertEquals(-1, index.tryReserveChildren(5, 32, 167 + (33 * 24) - 3)); + assertEquals(33, index.capacity()); + } + + @Test + public void testExpNanCapacityGrowth() { + var index = prepareIndex(167, 33); + double exp = 1.25 * index.capacity(); + assertEquals(33, index.tryReserveChildren(1, 0, 167)); + assertWithinRange((int)(0.95 * exp), (int)(1.05 * exp), index.capacity()); + } + + @Test + public void testMaxInfCapacityGrowth() { + var index = prepareIndex(167, 17); + double exp = 1.25 * index.capacity(); + assertEquals(17, index.tryReserveChildren(1, 10, 167)); + int maxSize = 10 + (17 * 24); + assertEquals(maxSize, index.capacity()); } } diff --git a/vespalib/src/vespa/vespalib/gtest/matchers/elements_are_distinct.h b/vespalib/src/vespa/vespalib/gtest/matchers/elements_are_distinct.h new file mode 100644 index 00000000000..374dafbf893 --- /dev/null +++ b/vespalib/src/vespa/vespalib/gtest/matchers/elements_are_distinct.h @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <gtest/gtest.h> +#include <gmock/gmock.h> +#include <ranges> + +/** + * Checks that all elements of a forward iterable range are distinct, i.e. the following must hold: + * - for any single element `foo`, foo == foo is true + * - for any two separate elements `foo` and `bar`, foo == bar is false + */ +MATCHER(ElementsAreDistinct, "") { + const auto& range = arg; + static_assert(std::ranges::forward_range<decltype(range)>); + const auto end = std::ranges::cend(range); + // Explicitly count element positions instead of comparing iterators to avoid depending + // on iterators being comparable with each other. + size_t i = 0; + for (auto lhs = std::ranges::cbegin(range); lhs != end; ++lhs, ++i) { + size_t j = 0; + for (auto rhs = std::ranges::cbegin(range); rhs != end; ++rhs, ++j) { + if (i != j) { + if (*lhs == *rhs) { + *result_listener << "Expected elements to be distinct, but element at position " + << i << " (" << *lhs << ") is equal to element at position " + << j << " (" << *rhs << ")"; + return false; + } + } else if (!(*lhs == *rhs)) { + *result_listener << "Element at position " << i << " (" << *lhs << ") does not equal itself"; + return false; + } + } + } + return true; +} diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 8ee3957af32..91365d446c1 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -31,6 +31,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT exceptions.cpp execution_profiler.cpp executor_idle_tracking.cpp + featureset.cpp file_area_freelist.cpp foregroundtaskexecutor.cpp gate.cpp diff --git a/searchlib/src/vespa/searchlib/common/featureset.cpp b/vespalib/src/vespa/vespalib/util/featureset.cpp index 5c8d4c6d9c4..6ac90461cfb 100644 --- a/searchlib/src/vespa/searchlib/common/featureset.cpp +++ b/vespalib/src/vespa/vespalib/util/featureset.cpp @@ -2,7 +2,7 @@ #include "featureset.h" -namespace search { +namespace vespalib { FeatureSet::FeatureSet() : _names(), @@ -87,4 +87,4 @@ FeatureSet::getFeaturesByDocId(uint32_t docId) const return 0; } -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/common/featureset.h b/vespalib/src/vespa/vespalib/util/featureset.h index adda8a2728b..ae7a0c6932f 100644 --- a/searchlib/src/vespa/searchlib/common/featureset.h +++ b/vespalib/src/vespa/vespalib/util/featureset.h @@ -2,14 +2,13 @@ #pragma once -#include "feature.h" #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/data/memory.h> #include <map> #include <vector> #include <memory> -namespace search { +namespace vespalib { /** * This class holds information about a set of features for a set of @@ -153,4 +152,4 @@ struct FeatureValues { std::vector<Value> values; // values.size() == names.size() * N }; -} // namespace search +} |