diff options
292 files changed, 3249 insertions, 9322 deletions
diff --git a/Gemfile.lock b/Gemfile.lock index e40e7b96017..b98be0cafcf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -5,16 +5,17 @@ GEM addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) afm (0.2.2) - async (2.6.4) + async (2.6.5) console (~> 1.10) fiber-annotation io-event (~> 1.1) timers (~> 4.1) colorator (1.1.0) concurrent-ruby (1.2.2) - console (1.23.2) + console (1.23.7) fiber-annotation fiber-local + json em-websocket (0.5.3) eventmachine (>= 0.12.9) http_parser.rb (~> 0) @@ -27,7 +28,7 @@ GEM forwardable-extended (2.6.0) google-protobuf (3.24.4-x86_64-linux) hashery (2.1.2) - html-proofer (5.0.8) + html-proofer (5.0.9) addressable (~> 2.3) async (~> 2.1) nokogiri (~> 1.13) @@ -39,7 +40,7 @@ GEM http_parser.rb (0.8.0) i18n (1.14.1) concurrent-ruby (~> 1.0) - io-event (1.3.2) + io-event (1.3.3) jekyll (4.3.3) addressable (~> 2.4) colorator (~> 1.0) @@ -64,6 +65,7 @@ GEM jekyll (>= 3.8, < 5.0) jekyll-watch (2.2.1) listen (~> 3.0) + json (2.7.2) kramdown (2.4.0) rexml kramdown-parser-gfm (1.1.0) @@ -77,7 +79,7 @@ GEM jekyll (>= 3.5, < 5.0) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) - nokogiri (1.16.2-x86_64-linux) + nokogiri (1.16.4-x86_64-linux) racc (~> 1.4) pathutil (0.16.2) forwardable-extended (~> 2.6) @@ -87,7 +89,7 @@ GEM hashery (~> 2.0) ruby-rc4 ttfunk - public_suffix (5.0.4) + public_suffix (5.0.5) racc (1.7.3) rainbow (3.1.1) rake (13.2.1) @@ -104,12 +106,12 @@ GEM unicode-display_width (>= 1.1.1, < 3) timers (4.3.5) ttfunk (1.7.0) - typhoeus (1.4.0) + typhoeus (1.4.1) ethon (>= 0.9.0) unicode-display_width (2.4.2) webrick (1.8.1) yell (2.2.2) - zeitwerk (2.6.11) + zeitwerk (2.6.13) PLATFORMS x86_64-linux diff --git a/bootstrap.sh b/bootstrap.sh index a98909799e1..43c62033f6d 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -37,13 +37,14 @@ get_env_var_with_optional_default() { } readonly MAVEN_CMD=$(get_env_var_with_optional_default VESPA_MAVEN_COMMAND "$(pwd)/mvnw") - readonly MAVEN_EXTRA_OPTS=$(get_env_var_with_optional_default VESPA_MAVEN_EXTRA_OPTS) +readonly MAVEN_TARGET=$(get_env_var_with_optional_default VESPA_MAVEN_TARGET "install") echo "Using maven command: ${MAVEN_CMD}" echo "Using maven extra opts: ${MAVEN_EXTRA_OPTS}" +echo "Using maven target: ${MAVEN_TARGET}" mvn_install() { - ${MAVEN_CMD} --batch-mode --no-snapshot-updates -Dmaven.wagon.http.retryHandler.count=5 clean install ${MAVEN_EXTRA_OPTS} "$@" + ${MAVEN_CMD} --batch-mode --no-snapshot-updates -Dmaven.wagon.http.retryHandler.count=5 clean ${MAVEN_TARGET} ${MAVEN_EXTRA_OPTS} "$@" } force_move() { @@ -105,7 +106,7 @@ case "$MODE" in ;; full) echo "Building full set of dependencies." - mvn_install -am -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -pl jrt,linguistics,messagebus + mvn_install -am -T1C -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -pl jrt,linguistics,messagebus ;; default) echo "Building default set of dependencies." diff --git a/client/go/go.mod b/client/go/go.mod index ba0af5a763e..7117d5ef334 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -16,8 +16,8 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/zalando/go-keyring v0.2.4 - golang.org/x/net v0.24.0 - golang.org/x/sys v0.19.0 + golang.org/x/net v0.25.0 + golang.org/x/sys v0.20.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -31,7 +31,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect - golang.org/x/term v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/client/go/go.sum b/client/go/go.sum index d985c9e7ffc..e98a14f862a 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -75,6 +75,8 @@ golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -91,6 +93,8 @@ golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= @@ -103,8 +107,12 @@ golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/client/go/internal/osutil/run_cmd.go b/client/go/internal/osutil/run_cmd.go index 3847dcc912a..ab7bd1069c3 100644 --- a/client/go/internal/osutil/run_cmd.go +++ b/client/go/internal/osutil/run_cmd.go @@ -38,7 +38,7 @@ func analyzeError(err error) string { msg := "died with signal: " + status.Signal().String() switch status.Signal() { case syscall.SIGILL: - msg = msg + " (you probably have an older CPU than required)" + msg = msg + " (you probably have an older CPU than required, see https://docs.vespa.ai/en/cpu-support.html)" } return msg } diff --git a/client/js/app/package.json b/client/js/app/package.json index 6de98422514..0725a768ab5 100644 --- a/client/js/app/package.json +++ b/client/js/app/package.json @@ -38,7 +38,7 @@ "prettier": "3", "pretty-quick": "^4.0.0", "react-router-dom": "^6", - "use-context-selector": "^1", + "use-context-selector": "^2.0.0", "vite": "^5.0.5" }, "jest": { diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 7074aa923ec..edeb568b30b 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -2,11 +2,6 @@ # yarn lockfile v1 -"@aashutoshrathi/word-wrap@^1.2.3": - version "1.2.6" - resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf" - integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA== - "@ampproject/remapping@^2.2.0": version "2.2.1" resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.1.tgz#99e8e11851128b8702cd57c33684f1d0f260b630" @@ -789,10 +784,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@9.1.1": - version "9.1.1" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.1.1.tgz#eb0f82461d12779bbafc1b5045cde3143d350a8a" - integrity sha512-5WoDz3Y19Bg2BnErkZTp0en+c/i9PvgFS7MBe1+m60HjFr0hrphlAGp4yzI7pxpt4xShln4ZyYp4neJm8hmOkQ== +"@eslint/js@9.2.0": + version "9.2.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.2.0.tgz#b0a9123e8e91a3d9a2eed3a04a6ed44fdab639aa" + integrity sha512-ESiIudvhoYni+MdsI8oD7skpprZ89qKocwRM2KEvhhBJ9nl5MRh7BXU5GTod7Mdygq+AUl+QzId6iWJKR/wABA== "@floating-ui/core@^1.4.2": version "1.5.0" @@ -883,9 +878,9 @@ integrity sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA== "@humanwhocodes/retry@^0.2.3": - version "0.2.3" - resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.2.3.tgz#c9aa036d1afa643f1250e83150f39efb3a15a631" - integrity sha512-X38nUbachlb01YMlvPFojKoiXq+LzZvuSce70KPMPdeM1Rj03k4dR7lDslhbqXn3Ang4EU3+EAmwEAsbrjHW3g== + version "0.2.4" + resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.2.4.tgz#4f3059423823bd8176132ceea9447dee101dfac1" + integrity sha512-Ttl/jHpxfS3st5sxwICYfk4pOH0WrLI1SpW283GgQL7sCWU7EHIOhX4b4fkIxr3tkfzwg8+FNojtzsIEE7Ecgg== "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" @@ -1325,85 +1320,85 @@ resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.16.0.tgz#0e10181e5fec1434eb071a9bc4bdaac843f16dcc" integrity sha512-Quz1KOffeEf/zwkCBM3kBtH4ZoZ+pT3xIXBG4PPW/XFtDP7EGhtTiC2+gpL9GnR7+Qdet5Oa6cYSvwKYg6kN9Q== -"@rollup/rollup-android-arm-eabi@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.15.0.tgz#28c9c79c5baccb59a96afcf60e428ea6965a5579" - integrity sha512-O63bJ7p909pRRQfOJ0k/Jp8gNFMud+ZzLLG5EBWquylHxmRT2k18M2ifg8WyjCgFVdpA7+rI0YZ8EkAtg6dSUw== - -"@rollup/rollup-android-arm64@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.15.0.tgz#a2bdafdb753ece571956289a5ba8c37af748bd0c" - integrity sha512-5UywPdmC9jiVOShjQx4uuIcnTQOf85iA4jgg8bkFoH5NYWFfAfrJpv5eeokmTdSmYwUTT5IrcrBCJNkowhrZDA== - -"@rollup/rollup-darwin-arm64@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.15.0.tgz#4cb44cfec3068f6f76f70463ccc25f3e245af06a" - integrity sha512-hNkt75uFfWpRxHItCBmbS0ba70WnibJh6yz60WShSWITLlVRbkvAu1E/c7RlliPY4ajhqJd0UPZz//gNalTd4g== - -"@rollup/rollup-darwin-x64@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.15.0.tgz#1035bfbf53e6acf16771191f41c3d3aff089e8f1" - integrity sha512-HnC5bTP7qdfO9nUw/mBhNcjOEZfbS8NwV+nFegiMhYOn1ATAGZF4kfAxR9BuZevBrebWCxMmxm8NCU1CUoz+wQ== - -"@rollup/rollup-linux-arm-gnueabihf@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.15.0.tgz#0036b835f17ca9e84c188c419399493bd5739986" - integrity sha512-QGOIQIJZeIIqMsc4BUGe8TnV4dkXhSW2EhaQ1G4LqMUNpkyeLztvlDlOoNHn7SR7a4dBANdcEbPkkEzz3rzjzA== - -"@rollup/rollup-linux-arm-musleabihf@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.15.0.tgz#c44420167203400ba7a707f8205413c2817cdaeb" - integrity sha512-PS/Cp8CinYgoysQ8i4UXYH/TZl06fXszvY/RDkyBYgUB1+tKyOMS925/4FZhfrhkl3XQEKjMc3BKtsxpB9Tz9Q== - -"@rollup/rollup-linux-arm64-gnu@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.15.0.tgz#531d3792e1526583ecd794ceee0ab980d79813dd" - integrity sha512-XzOsnD6lGDP+k+vGgTYAryVGu8N89qpjMN5BVFUj75dGVFP3FzIVAufJAraxirpDwEQZA7Gjs0Vo5p4UmnnjsA== - -"@rollup/rollup-linux-arm64-musl@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.15.0.tgz#86376eaa6d65a860a046e0dfe285a51792bc2026" - integrity sha512-+ScJA4Epbx/ZQGjDnbvTAcb8ZD06b+TlIka2UkujbKf1I/A+yrvEcJwG3/27zMmvcWMQyeCJhbL9TlSjzL0B7Q== - -"@rollup/rollup-linux-powerpc64le-gnu@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.15.0.tgz#6adf69ce27d1266dbb86eeac237ad5dd4d4a1f28" - integrity sha512-1cUSvYgnyTakM4FDyf/GxUCDcqmj/hUh1NOizEOJU7+D5xEfFGCxgcNOs3hYBeRMUCcGmGkt01EhD3ILgKpGHQ== - -"@rollup/rollup-linux-riscv64-gnu@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.15.0.tgz#961c290372d170f588ebf65c145c485c4aad7005" - integrity sha512-3A1FbHDbBUvpJXFAZwVsiROIcstVHP9AX/cwnyIhAp+xyQ1cBCxywKtuzmw0Av1MDNNg/y/9dDHtNypfRa8bdw== - -"@rollup/rollup-linux-s390x-gnu@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.15.0.tgz#cb43301e10f17f0a642416c5d3d82a26cf430fa8" - integrity sha512-hYPbhg9ow6/mXIkojc8LOeiip2sCTuw1taWyoOXTOWk9vawIXz8x7B4KkgWUAtvAElssxhSyEXr2EZycH/FGzQ== - -"@rollup/rollup-linux-x64-gnu@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.15.0.tgz#25c1fb87b2255949ee7ff6956e205710c5d7c414" - integrity sha512-511qln5mPSUKwv7HI28S1jCD1FK+2WbX5THM9A9annr3c1kzmfnf8Oe3ZakubEjob3IV6OPnNNcesfy+adIrmw== - -"@rollup/rollup-linux-x64-musl@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.15.0.tgz#145745e339e282c7afc36142bd5a3f9495c7c681" - integrity sha512-4qKKGTDIv2bQZ+afhPWqPL+94+dLtk4lw1iwbcylKlLNqQ/Yyjof2CFYBxf6npiDzPV+zf4EWRiHb26/4Vsm9w== - -"@rollup/rollup-win32-arm64-msvc@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.15.0.tgz#95dae687b645a25aab3a082d987556f58274ffbe" - integrity sha512-nEtaFBHp1OnbOf+tz66DtID579sNRHGgMC23to8HUyVuOCpCMD0CvRNqiDGLErLNnwApWIUtUl1VvuovCWUxwg== - -"@rollup/rollup-win32-ia32-msvc@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.15.0.tgz#acbd48f10093e6cd52f99ad004966433a49cb362" - integrity sha512-5O49NykwSgX6iT2HgZ6cAoGHt6T/FqNMB5OqFOGxU/y1GyFSHquox1sK2OqApQc0ANxiHFQEMNDLNVCL7AUDnQ== - -"@rollup/rollup-win32-x64-msvc@4.15.0": - version "4.15.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.15.0.tgz#68bb584231dfc8e36bb7ad5317dfd1fd2563a6f7" - integrity sha512-YA0hTwCunmKNeTOFWdJuKhdXse9jBqgo34FDo+9aS0spfCkp+wj0o1bCcOOTu+0P48O95GTfkLTAaVonwNuIdQ== +"@rollup/rollup-android-arm-eabi@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.17.2.tgz#1a32112822660ee104c5dd3a7c595e26100d4c2d" + integrity sha512-NM0jFxY8bB8QLkoKxIQeObCaDlJKewVlIEkuyYKm5An1tdVZ966w2+MPQ2l8LBZLjR+SgyV+nRkTIunzOYBMLQ== + +"@rollup/rollup-android-arm64@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.17.2.tgz#5aeef206d65ff4db423f3a93f71af91b28662c5b" + integrity sha512-yeX/Usk7daNIVwkq2uGoq2BYJKZY1JfyLTaHO/jaiSwi/lsf8fTFoQW/n6IdAsx5tx+iotu2zCJwz8MxI6D/Bw== + +"@rollup/rollup-darwin-arm64@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.17.2.tgz#6b66aaf003c70454c292cd5f0236ebdc6ffbdf1a" + integrity sha512-kcMLpE6uCwls023+kknm71ug7MZOrtXo+y5p/tsg6jltpDtgQY1Eq5sGfHcQfb+lfuKwhBmEURDga9N0ol4YPw== + +"@rollup/rollup-darwin-x64@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.17.2.tgz#f64fc51ed12b19f883131ccbcea59fc68cbd6c0b" + integrity sha512-AtKwD0VEx0zWkL0ZjixEkp5tbNLzX+FCqGG1SvOu993HnSz4qDI6S4kGzubrEJAljpVkhRSlg5bzpV//E6ysTQ== + +"@rollup/rollup-linux-arm-gnueabihf@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.17.2.tgz#1a7641111be67c10111f7122d1e375d1226cbf14" + integrity sha512-3reX2fUHqN7sffBNqmEyMQVj/CKhIHZd4y631duy0hZqI8Qoqf6lTtmAKvJFYa6bhU95B1D0WgzHkmTg33In0A== + +"@rollup/rollup-linux-arm-musleabihf@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.17.2.tgz#c93fd632923e0fee25aacd2ae414288d0b7455bb" + integrity sha512-uSqpsp91mheRgw96xtyAGP9FW5ChctTFEoXP0r5FAzj/3ZRv3Uxjtc7taRQSaQM/q85KEKjKsZuiZM3GyUivRg== + +"@rollup/rollup-linux-arm64-gnu@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.17.2.tgz#fa531425dd21d058a630947527b4612d9d0b4a4a" + integrity sha512-EMMPHkiCRtE8Wdk3Qhtciq6BndLtstqZIroHiiGzB3C5LDJmIZcSzVtLRbwuXuUft1Cnv+9fxuDtDxz3k3EW2A== + +"@rollup/rollup-linux-arm64-musl@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.17.2.tgz#8acc16f095ceea5854caf7b07e73f7d1802ac5af" + integrity sha512-NMPylUUZ1i0z/xJUIx6VUhISZDRT+uTWpBcjdv0/zkp7b/bQDF+NfnfdzuTiB1G6HTodgoFa93hp0O1xl+/UbA== + +"@rollup/rollup-linux-powerpc64le-gnu@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.17.2.tgz#94e69a8499b5cf368911b83a44bb230782aeb571" + integrity sha512-T19My13y8uYXPw/L/k0JYaX1fJKFT/PWdXiHr8mTbXWxjVF1t+8Xl31DgBBvEKclw+1b00Chg0hxE2O7bTG7GQ== + +"@rollup/rollup-linux-riscv64-gnu@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.17.2.tgz#7ef1c781c7e59e85a6ce261cc95d7f1e0b56db0f" + integrity sha512-BOaNfthf3X3fOWAB+IJ9kxTgPmMqPPH5f5k2DcCsRrBIbWnaJCgX2ll77dV1TdSy9SaXTR5iDXRL8n7AnoP5cg== + +"@rollup/rollup-linux-s390x-gnu@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.17.2.tgz#f15775841c3232fca9b78cd25a7a0512c694b354" + integrity sha512-W0UP/x7bnn3xN2eYMql2T/+wpASLE5SjObXILTMPUBDB/Fg/FxC+gX4nvCfPBCbNhz51C+HcqQp2qQ4u25ok6g== + +"@rollup/rollup-linux-x64-gnu@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.17.2.tgz#b521d271798d037ad70c9f85dd97d25f8a52e811" + integrity sha512-Hy7pLwByUOuyaFC6mAr7m+oMC+V7qyifzs/nW2OJfC8H4hbCzOX07Ov0VFk/zP3kBsELWNFi7rJtgbKYsav9QQ== + +"@rollup/rollup-linux-x64-musl@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.17.2.tgz#9254019cc4baac35800991315d133cc9fd1bf385" + integrity sha512-h1+yTWeYbRdAyJ/jMiVw0l6fOOm/0D1vNLui9iPuqgRGnXA0u21gAqOyB5iHjlM9MMfNOm9RHCQ7zLIzT0x11Q== + +"@rollup/rollup-win32-arm64-msvc@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.17.2.tgz#27f65a89f6f52ee9426ec11e3571038e4671790f" + integrity sha512-tmdtXMfKAjy5+IQsVtDiCfqbynAQE/TQRpWdVataHmhMb9DCoJxp9vLcCBjEQWMiUYxO1QprH/HbY9ragCEFLA== + +"@rollup/rollup-win32-ia32-msvc@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.17.2.tgz#a2fbf8246ed0bb014f078ca34ae6b377a90cb411" + integrity sha512-7II/QCSTAHuE5vdZaQEwJq2ZACkBpQDOmQsE6D6XUbnBHW8IAhm4eTufL6msLJorzrHDFv3CF8oCA/hSIRuZeQ== + +"@rollup/rollup-win32-x64-msvc@4.17.2": + version "4.17.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.17.2.tgz#5a2d08b81e8064b34242d5cc9973ef8dd1e60503" + integrity sha512-TGGO7v7qOq4CYmSBVEYpI1Y5xDuCEnbVC5Vth8mOsW0gDSzxNrVERPc790IGHsrT2dQSimgMr9Ub3Y1Jci5/8w== "@sinclair/typebox@^0.27.8": version "0.27.8" @@ -2605,9 +2600,9 @@ eslint-plugin-prettier@^5: synckit "^0.8.6" eslint-plugin-react-hooks@^4: - version "4.6.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.0.tgz#4c3e697ad95b77e93f8646aaa1630c1ba607edd3" - integrity sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g== + version "4.6.2" + resolved "https://registry.yarnpkg.com/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.2.tgz#c829eb06c0e6f484b3fbb85a97e57784f328c596" + integrity sha512-QzliNJq4GinDBcD8gPB5v0wh6g8q3SUi6EFF0x8N/BL9PoVs0atuGc47ozMRyOWAKdwaZ5OnbOEa3WR+dSGKuQ== eslint-plugin-react-perf@^3: version "3.3.2" @@ -2639,9 +2634,9 @@ eslint-plugin-react@^7: string.prototype.matchall "^4.0.10" eslint-plugin-unused-imports@^3: - version "3.1.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-unused-imports/-/eslint-plugin-unused-imports-3.1.0.tgz#db015b569d3774e17a482388c95c17bd303bc602" - integrity sha512-9l1YFCzXKkw1qtAru1RWUtG2EVDZY0a0eChKXcL+EZ5jitG7qxdctu4RnvhOJHv4xfmUf7h+JJPINlVpGhZMrw== + version "3.2.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-unused-imports/-/eslint-plugin-unused-imports-3.2.0.tgz#63a98c9ad5f622cd9f830f70bc77739f25ccfe0d" + integrity sha512-6uXyn6xdINEpxE1MtDjxQsyXB37lfyO2yKGVVgtD7WEWQGORSOZjgrD6hBhvGv4/SO+TOlS+UnC6JppRqbuwGQ== dependencies: eslint-rule-composer "^0.3.0" @@ -2669,14 +2664,14 @@ eslint-visitor-keys@^4.0.0: integrity sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw== eslint@^9.0.0: - version "9.1.1" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.1.1.tgz#39ec657ccd12813cb4a1dab2f9229dcc6e468271" - integrity sha512-b4cRQ0BeZcSEzPpY2PjFY70VbO32K7BStTGtBsnIGdTSEEQzBi8hPBcGQmTG2zUvFr9uLe0TK42bw8YszuHEqg== + version "9.2.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.2.0.tgz#0700ebc99528753315d78090876911d3cdbf19fe" + integrity sha512-0n/I88vZpCOzO+PQpt0lbsqmn9AsnsJAQseIqhZFI8ibQT0U1AkEKRxA3EVMos0BoHSXDQvCXY25TUjB5tr8Og== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.6.1" "@eslint/eslintrc" "^3.0.2" - "@eslint/js" "9.1.1" + "@eslint/js" "9.2.0" "@humanwhocodes/config-array" "^0.13.0" "@humanwhocodes/module-importer" "^1.0.1" "@humanwhocodes/retry" "^0.2.3" @@ -4473,16 +4468,16 @@ onetime@^5.1.2: mimic-fn "^2.1.0" optionator@^0.9.3: - version "0.9.3" - resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.3.tgz#007397d44ed1872fdc6ed31360190f81814e2c64" - integrity sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg== + version "0.9.4" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.4.tgz#7ea1c1a5d91d764fb282139c88fe11e182a3a734" + integrity sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g== dependencies: - "@aashutoshrathi/word-wrap" "^1.2.3" deep-is "^0.1.3" fast-levenshtein "^2.0.6" levn "^0.4.1" prelude-ls "^1.2.1" type-check "^0.4.0" + word-wrap "^1.2.5" p-finally@^1.0.0: version "1.0.0" @@ -4700,12 +4695,12 @@ queue-microtask@^1.2.2: integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== react-dom@^18: - version "18.2.0" - resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-18.2.0.tgz#22aaf38708db2674ed9ada224ca4aa708d821e3d" - integrity sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g== + version "18.3.1" + resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-18.3.1.tgz#c2265d79511b57d479b3dd3fdfa51536494c5cb4" + integrity sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw== dependencies: loose-envify "^1.1.0" - scheduler "^0.23.0" + scheduler "^0.23.2" react-is@^16.13.1, react-is@^16.7.0: version "16.13.1" @@ -4757,9 +4752,9 @@ react-transition-group@4.4.2: prop-types "^15.6.2" react@^18: - version "18.2.0" - resolved "https://registry.yarnpkg.com/react/-/react-18.2.0.tgz#555bd98592883255fa00de14f1151a917b5d77d5" - integrity sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ== + version "18.3.1" + resolved "https://registry.yarnpkg.com/react/-/react-18.3.1.tgz#49ab892009c53933625bd16b2533fc754cab2891" + integrity sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ== dependencies: loose-envify "^1.1.0" @@ -4884,28 +4879,28 @@ reusify@^1.0.4: integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== rollup@^4.13.0: - version "4.15.0" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.15.0.tgz#3be428e4fe86297b1b3448f29515d978593d9d9a" - integrity sha512-i0ir57IMF5o7YvNYyUNeIGG+IZaaucnGZAOsSctO2tPLXlCEaZzyBa+QhpHNSgtpyLMoDev2DyN6a7J1dQA8Tw== + version "4.17.2" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.17.2.tgz#26d1785d0144122277fdb20ab3a24729ae68301f" + integrity sha512-/9ClTJPByC0U4zNLowV1tMBe8yMEAxewtR3cUNX5BoEpGH3dQEWpJLr6CLp0fPdYRF/fzVOgvDb1zXuakwF5kQ== dependencies: "@types/estree" "1.0.5" optionalDependencies: - "@rollup/rollup-android-arm-eabi" "4.15.0" - "@rollup/rollup-android-arm64" "4.15.0" - "@rollup/rollup-darwin-arm64" "4.15.0" - "@rollup/rollup-darwin-x64" "4.15.0" - "@rollup/rollup-linux-arm-gnueabihf" "4.15.0" - "@rollup/rollup-linux-arm-musleabihf" "4.15.0" - "@rollup/rollup-linux-arm64-gnu" "4.15.0" - "@rollup/rollup-linux-arm64-musl" "4.15.0" - "@rollup/rollup-linux-powerpc64le-gnu" "4.15.0" - "@rollup/rollup-linux-riscv64-gnu" "4.15.0" - "@rollup/rollup-linux-s390x-gnu" "4.15.0" - "@rollup/rollup-linux-x64-gnu" "4.15.0" - "@rollup/rollup-linux-x64-musl" "4.15.0" - "@rollup/rollup-win32-arm64-msvc" "4.15.0" - "@rollup/rollup-win32-ia32-msvc" "4.15.0" - "@rollup/rollup-win32-x64-msvc" "4.15.0" + "@rollup/rollup-android-arm-eabi" "4.17.2" + "@rollup/rollup-android-arm64" "4.17.2" + "@rollup/rollup-darwin-arm64" "4.17.2" + "@rollup/rollup-darwin-x64" "4.17.2" + "@rollup/rollup-linux-arm-gnueabihf" "4.17.2" + "@rollup/rollup-linux-arm-musleabihf" "4.17.2" + "@rollup/rollup-linux-arm64-gnu" "4.17.2" + "@rollup/rollup-linux-arm64-musl" "4.17.2" + "@rollup/rollup-linux-powerpc64le-gnu" "4.17.2" + "@rollup/rollup-linux-riscv64-gnu" "4.17.2" + "@rollup/rollup-linux-s390x-gnu" "4.17.2" + "@rollup/rollup-linux-x64-gnu" "4.17.2" + "@rollup/rollup-linux-x64-musl" "4.17.2" + "@rollup/rollup-win32-arm64-msvc" "4.17.2" + "@rollup/rollup-win32-ia32-msvc" "4.17.2" + "@rollup/rollup-win32-x64-msvc" "4.17.2" fsevents "~2.3.2" rsvp@^4.8.4: @@ -4961,10 +4956,10 @@ sane@^4.0.3: minimist "^1.1.1" walker "~1.0.5" -scheduler@^0.23.0: - version "0.23.0" - resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.23.0.tgz#ba8041afc3d30eb206a487b6b384002e4e61fdfe" - integrity sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw== +scheduler@^0.23.2: + version "0.23.2" + resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.23.2.tgz#414ba64a3b282892e944cf2108ecc078d115cdc3" + integrity sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ== dependencies: loose-envify "^1.1.0" @@ -5489,10 +5484,10 @@ use-composed-ref@^1.3.0: resolved "https://registry.yarnpkg.com/use-composed-ref/-/use-composed-ref-1.3.0.tgz#3d8104db34b7b264030a9d916c5e94fbe280dbda" integrity sha512-GLMG0Jc/jiKov/3Ulid1wbv3r54K9HlMW29IWcDFPEqFkSO2nS0MuefWgMJpeHQ9YJeXDL3ZUF+P3jdXlZX/cQ== -use-context-selector@^1: - version "1.4.4" - resolved "https://registry.yarnpkg.com/use-context-selector/-/use-context-selector-1.4.4.tgz#f5d65c7fcd78f994cb33cacd57651007a40595c0" - integrity sha512-pS790zwGxxe59GoBha3QYOwk8AFGp4DN6DOtH+eoqVmgBBRXVx4IlPDhJmmMiNQAgUaLlP+58aqRC3A4rdaSjg== +use-context-selector@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/use-context-selector/-/use-context-selector-2.0.0.tgz#3b5dafec7aa947c152d4f0aa7f250e99a205df3d" + integrity sha512-owfuSmUNd3eNp3J9CdDl0kMgfidV+MkDvHPpvthN5ThqM+ibMccNE0k+Iq7TWC6JPFvGZqanqiGCuQx6DyV24g== use-isomorphic-layout-effect@^1.1.1: version "1.1.2" @@ -5521,9 +5516,9 @@ v8-to-istanbul@^9.0.1: convert-source-map "^1.6.0" vite@^5.0.5: - version "5.2.10" - resolved "https://registry.yarnpkg.com/vite/-/vite-5.2.10.tgz#2ac927c91e99d51b376a5c73c0e4b059705f5bd7" - integrity sha512-PAzgUZbP7msvQvqdSD+ErD5qGnSFiGOoWmV5yAKUEI0kdhjbH6nMWVyZQC/hSc4aXwc0oJ9aEdIiF9Oje0JFCw== + version "5.2.11" + resolved "https://registry.yarnpkg.com/vite/-/vite-5.2.11.tgz#726ec05555431735853417c3c0bfb36003ca0cbd" + integrity sha512-HndV31LWW05i1BLPMUCE1B9E9GFbOu1MbenhS58FuK6owSO5qHm7GiCotrNY1YE5rMeQSFBGmT5ZaLEjFizgiQ== dependencies: esbuild "^0.20.1" postcss "^8.4.38" @@ -5602,6 +5597,11 @@ which@^2.0.1: dependencies: isexe "^2.0.0" +word-wrap@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.5.tgz#d2c45c6dd4fbce621a66f136cbe328afd0410b34" + integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA== + wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AggregatedClusterStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AggregatedClusterStats.java index 37698a3ad00..aa2a1d29ec0 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AggregatedClusterStats.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AggregatedClusterStats.java @@ -10,4 +10,6 @@ public interface AggregatedClusterStats { ContentClusterStats getStats(); + ContentNodeStats getGlobalStats(); + } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java index f1c19bac9b6..6fb31cc1b1c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java @@ -38,6 +38,9 @@ public class ClusterStatsAggregator { // Maps the content node index to the content node stats for that node. // This MUST be kept up-to-date with distributorToStats; private final ContentClusterStats aggregatedStats; + // This is the aggregate of aggregates across content nodes, allowing a reader to + // get a O(1) view of all merges pending in the cluster. + private final ContentNodeStats globallyAggregatedNodeStats = new ContentNodeStats(-1); ClusterStatsAggregator(Set<Integer> distributors, Set<Integer> storageNodes) { this.distributors = distributors; @@ -58,6 +61,10 @@ public class ClusterStatsAggregator { return aggregatedStats; } + @Override + public ContentNodeStats getGlobalStats() { + return globallyAggregatedNodeStats; + } }; } @@ -96,12 +103,14 @@ public class ClusterStatsAggregator { ContentNodeStats statsToAdd = clusterStats.getNodeStats(nodeIndex); if (statsToAdd != null) { contentNode.add(statsToAdd); + globallyAggregatedNodeStats.add(statsToAdd); } if (prevClusterStats != null) { ContentNodeStats statsToSubtract = prevClusterStats.getNodeStats(nodeIndex); if (statsToSubtract != null) { contentNode.subtract(statsToSubtract); + globallyAggregatedNodeStats.subtract(statsToSubtract); } } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 3e520d95d2c..3f7214c31e2 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -542,6 +542,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta didWork |= metricUpdater.forWork("processNextQueuedRemoteTask", this::processNextQueuedRemoteTask); didWork |= metricUpdater.forWork("completeSatisfiedVersionDependentTasks", this::completeSatisfiedVersionDependentTasks); didWork |= metricUpdater.forWork("maybePublishOldMetrics", this::maybePublishOldMetrics); + updateClusterSyncMetrics(); processingCycle = false; ++cycleCount; @@ -563,6 +564,14 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta } } + private void updateClusterSyncMetrics() { + var stats = stateVersionTracker.getAggregatedClusterStats().getAggregatedStats(); + if (stats.hasUpdatesFromAllDistributors()) { + GlobalBucketSyncStatsCalculator.clusterBucketsOutOfSyncRatio(stats.getGlobalStats()) + .ifPresent(metricUpdater::updateClusterBucketsOutOfSyncRatio); + } + } + private boolean updateMasterElectionState() { try { return masterElectionHandler.watchMasterElection(database, databaseContext); @@ -689,6 +698,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta context.cluster = cluster; context.currentConsolidatedState = consolidatedClusterState(); context.publishedClusterStateBundle = stateVersionTracker.getVersionedClusterStateBundle(); + context.aggregatedClusterStats = stateVersionTracker.getAggregatedClusterStats().getAggregatedStats(); context.masterInfo = new MasterInterface() { @Override public boolean isMaster() { return isMaster; } @Override public Integer getMaster() { return masterElectionHandler.getMaster(); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/GlobalBucketSyncStatsCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/GlobalBucketSyncStatsCalculator.java new file mode 100644 index 00000000000..0137ea2c29e --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/GlobalBucketSyncStatsCalculator.java @@ -0,0 +1,45 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import java.util.Optional; + +/** + * @author vekterli + */ +public class GlobalBucketSyncStatsCalculator { + + /** + * Compute a value in [0, 1] representing how much of the cluster's data space is currently + * out of sync, i.e. pending merging. In other words, if the value is 1 all buckets are out + * of sync, and conversely if it's 0 all buckets are in sync. This number applies across bucket + * spaces. + * + * @param globalStats Globally aggregated content node statistics for the entire cluster. + * @return Optional containing a value [0, 1] representing the ratio of buckets pending merge + * in relation to the total number of buckets in the cluster, or an empty optional if + * the underlying global statistics contains invalid/incomplete information. + */ + public static Optional<Double> clusterBucketsOutOfSyncRatio(ContentNodeStats globalStats) { + long totalBuckets = 0; + long pendingBuckets = 0; + for (var space : globalStats.getBucketSpaces().values()) { + if (!space.valid()) { + return Optional.empty(); + } + totalBuckets += space.getBucketsTotal(); + pendingBuckets += space.getBucketsPending(); + } + // It's currently possible for the reported number of pending buckets to be greater than + // the number of total buckets. Example: this can happen if a bucket is present on a single + // node, but should have been replicated to 9 more nodes. Since counts are not normalized + // across content nodes for a given bucket, this will be counted as 9 pending and 1 total. + // Eventually this will settle as 0 pending and 10 total. + // TODO report node-normalized pending/total counts from distributors and use these. + pendingBuckets = Math.min(pendingBuckets, totalBuckets); + if (totalBuckets <= 0) { + return Optional.of(0.0); // No buckets; cannot be out of sync by definition + } + return Optional.of((double)pendingBuckets / (double)totalBuckets); + } + +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java index 419cb652671..d149d4043e4 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java @@ -93,6 +93,10 @@ public class MetricUpdater { metricReporter.set("is-master", isMaster ? 1 : 0); } + public void updateClusterBucketsOutOfSyncRatio(double ratio) { + metricReporter.set("cluster-buckets-out-of-sync-ratio", ratio); + } + public void addTickTime(long millis, boolean didWork) { if (didWork) { metricReporter.set("busy-tick-time-ms", millis); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java index efb161cebec..e1b774e64ff 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java @@ -17,6 +17,7 @@ public abstract class RemoteClusterControllerTask { public MasterInterface masterInfo; public NodeListener nodeListener; public SlobrokListener slobrokListener; + public AggregatedClusterStats aggregatedClusterStats; } private final Object monitor = new Object(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java index 636d01dbfa3..7af5f93fa21 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java @@ -76,7 +76,7 @@ public class Response { { protected final Map<String, String> attributes = new LinkedHashMap<>(); protected final Map<String, SubUnitList> subUnits = new LinkedHashMap<>(); - protected final Map<String, Long> metrics = new LinkedHashMap<>(); + protected final Map<String, Number> metrics = new LinkedHashMap<>(); protected final Map<String, UnitState> stateMap = new LinkedHashMap<>(); protected DistributionState publishedState = null; @@ -94,7 +94,7 @@ public class Response { } @Override - public Map<String, Long> getMetricMap() { return metrics; } + public Map<String, Number> getMetricMap() { return metrics; } @Override public Map<String, UnitState> getStatePerType() { return stateMap; } @Override @@ -122,7 +122,7 @@ public class Response { list.addUnit(unit, response); return this; } - public EmptyResponse<T> addMetric(String name, Long value) { + public EmptyResponse<T> addMetric(String name, Number value) { metrics.put(name, value); return this; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/ClusterStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/ClusterStateRequest.java index 1df37637dcf..3006effecd4 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/ClusterStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/ClusterStateRequest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.clustercontroller.core.restapiv2.requests; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; +import com.yahoo.vespa.clustercontroller.core.GlobalBucketSyncStatsCalculator; import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTask; import com.yahoo.vespa.clustercontroller.core.restapiv2.Id; import com.yahoo.vespa.clustercontroller.core.restapiv2.Request; @@ -36,6 +37,11 @@ public class ClusterStateRequest extends Request<Response.ClusterResponse> { } } result.setPublishedState(bundleToDistributionState(context.publishedClusterStateBundle)); + if (context.aggregatedClusterStats.hasUpdatesFromAllDistributors()) { + var stats = context.aggregatedClusterStats.getGlobalStats(); + var maybeRatio = GlobalBucketSyncStatsCalculator.clusterBucketsOutOfSyncRatio(stats); + maybeRatio.ifPresent(r -> result.addMetric("cluster-buckets-out-of-sync-ratio", r)); + } return result; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java index aa47ce2ec82..14276c51416 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java @@ -33,6 +33,10 @@ public class ClusterStatsAggregatorTest { assertEquals(expectedStats.build(), aggregator.getAggregatedStatsForDistributor(distributorIndex)); } + public void verifyGlobal(ContentNodeStatsBuilder expectedStats) { + assertEquals(expectedStats.build(), aggregator.getAggregatedStats().getGlobalStats()); + } + boolean hasUpdatesFromAllDistributors() { return aggregator.getAggregatedStats().hasUpdatesFromAllDistributors(); } @@ -64,6 +68,10 @@ public class ClusterStatsAggregatorTest { return Sets.newHashSet(indices); } + private static ContentNodeStatsBuilder globalStatsBuilder() { + return ContentNodeStatsBuilder.forNode(-1); + } + @Test void aggregator_handles_updates_to_single_distributor_and_content_node() { Fixture f = new Fixture(distributorNodes(1), contentNodes(3)); @@ -72,6 +80,9 @@ public class ClusterStatsAggregatorTest { .add(3, "global", 11, 2); f.update(1, stats); f.verify(stats); + f.verifyGlobal(globalStatsBuilder() + .add("default", 10, 1) + .add("global", 11, 2)); } @Test @@ -80,9 +91,13 @@ public class ClusterStatsAggregatorTest { f.verify(new ContentClusterStatsBuilder() .add(3, "default", 10 + 14, 1 + 5) - .add(3, "global", 11 + 15, 2 + 6) + .add(3, "global", 11 + 15, 2 + 6) .add(4, "default", 12 + 16, 3 + 7) - .add(4, "global", 13 + 17, 4 + 8)); + .add(4, "global", 13 + 17, 4 + 8)); + + f.verifyGlobal(globalStatsBuilder() + .add("default", (10 + 14) + (12 + 16), (1 + 5) + (3 + 7)) + .add("global", (11 + 15) + (13 + 17), (2 + 6) + (4 + 8))); } @Test @@ -94,28 +109,34 @@ public class ClusterStatsAggregatorTest { f.update(2, new ContentClusterStatsBuilder().add(3, "default", 10, 1)); f.verify(new ContentClusterStatsBuilder().addInvalid(3, "default", 10, 1)); + f.verifyGlobal(globalStatsBuilder().addInvalid("default", 10, 1)); f.update(1, new ContentClusterStatsBuilder().add(3, "default", 11, 2)); f.verify(new ContentClusterStatsBuilder().add(3, "default", 10 + 11, 1 + 2)); + f.verifyGlobal(globalStatsBuilder().add("default", 10 + 11, 1 + 2)); f.update(2, new ContentClusterStatsBuilder().add(3, "default", 15, 6)); f.verify(new ContentClusterStatsBuilder().add(3, "default", 11 + 15, 2 + 6)); + f.verifyGlobal(globalStatsBuilder().add("default", 11 + 15, 2 + 6)); f.update(1, new ContentClusterStatsBuilder().add(3, "default", 16, 7)); f.verify(new ContentClusterStatsBuilder().add(3, "default", 15 + 16, 6 + 7)); + f.verifyGlobal(globalStatsBuilder().add("default", 15 + 16, 6 + 7)); f.update(2, new ContentClusterStatsBuilder().add(3, "default", 12, 3)); f.verify(new ContentClusterStatsBuilder().add(3, "default", 16 + 12, 7 + 3)); + f.verifyGlobal(globalStatsBuilder().add("default", 16 + 12, 7 + 3)); } @Test - void aggregator_handles_more_content_nodes_that_distributors() { + void aggregator_handles_more_content_nodes_than_distributors() { Fixture f = new Fixture(distributorNodes(1), contentNodes(3, 4)); ContentClusterStatsBuilder stats = new ContentClusterStatsBuilder() .add(3, "default", 10, 1) .add(4, "default", 11, 2); f.update(1, stats); f.verify(stats); + f.verifyGlobal(globalStatsBuilder().add("default", 10 + 11, 1 + 2)); } @Test diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStatsBuilder.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStatsBuilder.java index 9d4664a9362..34035793e75 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStatsBuilder.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStatsBuilder.java @@ -13,7 +13,7 @@ public class ContentNodeStatsBuilder { this.nodeIndex = nodeIndex; } - static ContentNodeStatsBuilder forNode(int nodeIndex) { + public static ContentNodeStatsBuilder forNode(int nodeIndex) { return new ContentNodeStatsBuilder(nodeIndex); } @@ -21,12 +21,16 @@ public class ContentNodeStatsBuilder { return add(bucketSpace, ContentNodeStats.BucketSpaceStats.of(bucketsTotal, bucketsPending)); } + public ContentNodeStatsBuilder addInvalid(String bucketSpace, long bucketsTotal, long bucketsPending) { + return add(bucketSpace, ContentNodeStats.BucketSpaceStats.invalid(bucketsTotal, bucketsPending)); + } + public ContentNodeStatsBuilder add(String bucketSpace, ContentNodeStats.BucketSpaceStats bucketSpaceStats) { stats.put(bucketSpace, bucketSpaceStats); return this; } - ContentNodeStats build() { + public ContentNodeStats build() { return new ContentNodeStats(nodeIndex, stats); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GlobalBucketSyncStatsCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GlobalBucketSyncStatsCalculatorTest.java new file mode 100644 index 00000000000..d44aaa54a1d --- /dev/null +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GlobalBucketSyncStatsCalculatorTest.java @@ -0,0 +1,59 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class GlobalBucketSyncStatsCalculatorTest { + + private static ContentNodeStatsBuilder globalStatsBuilder() { + return ContentNodeStatsBuilder.forNode(-1); + } + + private static void assertComputedRatio(double expected, ContentNodeStatsBuilder statsBuilder) { + var maybeRatio = GlobalBucketSyncStatsCalculator.clusterBucketsOutOfSyncRatio(statsBuilder.build()); + if (maybeRatio.isEmpty()) { + throw new IllegalArgumentException("Expected calculation to yield a value, but was empty"); + } + assertEquals(expected, maybeRatio.get(), 0.00001); + } + + private static void assertEmptyComputedRatio(ContentNodeStatsBuilder statsBuilder) { + var maybeRatio = GlobalBucketSyncStatsCalculator.clusterBucketsOutOfSyncRatio(statsBuilder.build()); + assertTrue(maybeRatio.isEmpty()); + } + + @Test + void no_buckets_imply_fully_in_sync() { + // Can't have anything out of sync if you don't have anything to be out of sync with *taps side of head* + assertComputedRatio(0.0, globalStatsBuilder().add("default", 0, 0)); + } + + @Test + void no_pending_buckets_implies_fully_in_sync() { + assertComputedRatio(0.0, globalStatsBuilder().add("default", 100, 0)); + assertComputedRatio(0.0, globalStatsBuilder().add("default", 100, 0).add("global", 50, 0)); + } + + @Test + void invalid_stats_returns_empty() { + assertEmptyComputedRatio(globalStatsBuilder().add("default", ContentNodeStats.BucketSpaceStats.invalid())); + assertEmptyComputedRatio(globalStatsBuilder() + .add("default", 100, 0) + .add("global", ContentNodeStats.BucketSpaceStats.invalid())); + } + + @Test + void pending_buckets_return_expected_ratio() { + assertComputedRatio(0.50, globalStatsBuilder().add("default", 10, 5)); + assertComputedRatio(0.80, globalStatsBuilder().add("default", 10, 8)); + assertComputedRatio(0.10, globalStatsBuilder().add("default", 100, 10)); + assertComputedRatio(0.01, globalStatsBuilder().add("default", 100, 1)); + assertComputedRatio(0.05, globalStatsBuilder().add("default", 50, 5).add("global", 50, 0)); + assertComputedRatio(0.05, globalStatsBuilder().add("default", 50, 0).add("global", 50, 5)); + assertComputedRatio(0.10, globalStatsBuilder().add("default", 50, 5).add("global", 50, 5)); + } + +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java index d06cc730b3f..902b1bce24a 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java @@ -15,6 +15,8 @@ public class ClusterControllerMock implements RemoteClusterControllerTaskSchedul private final int fleetControllerIndex; Integer fleetControllerMaster; private final StringBuilder events = new StringBuilder(); + ContentNodeStats globalClusterStats = new ContentNodeStats(-1); + boolean enableGlobalStatsReporting = false; ClusterControllerMock(ContentCluster cluster, ClusterState state, ClusterStateBundle publishedClusterStateBundle, @@ -88,6 +90,22 @@ public class ClusterControllerMock implements RemoteClusterControllerTaskSchedul } }; + context.aggregatedClusterStats = new AggregatedClusterStats() { + @Override + public boolean hasUpdatesFromAllDistributors() { + return enableGlobalStatsReporting; + } + + @Override + public ContentClusterStats getStats() { + return null; + } + + @Override + public ContentNodeStats getGlobalStats() { + return globalClusterStats; + } + }; } @Override diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java index e4b3c0b9f2c..cb1213542ce 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.restapiv2; +import com.yahoo.vespa.clustercontroller.core.ContentNodeStatsBuilder; import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitResponse; import org.junit.jupiter.api.Test; @@ -105,4 +106,45 @@ public class ClusterTest extends StateRestApiTest { }""", jsonWriter.createJson(response).toPrettyString()); } + + @Test + void emit_cluster_stats_if_present() throws Exception { + setUp(true); + books.globalClusterStats.add(ContentNodeStatsBuilder.forNode(-1).add("default", 10, 4).build()); + books.enableGlobalStatsReporting = true; + UnitResponse response = restAPI.getState(new StateRequest("books", 0)); + assertEquals(""" + { + "state" : { + "generated" : { + "state" : "up", + "reason" : "" + } + }, + "metrics" : { + "cluster-buckets-out-of-sync-ratio" : 0.4 + }, + "service" : { + "storage" : { + "link" : "/cluster/v2/books/storage" + }, + "distributor" : { + "link" : "/cluster/v2/books/distributor" + } + }, + "distribution-states" : { + "published" : { + "baseline" : "distributor:4 storage:4", + "bucket-spaces" : [ { + "name" : "default", + "state" : "distributor:4 storage:4 .3.s:m" + }, { + "name" : "global", + "state" : "distributor:4 storage:4" + } ] + } + } + }""", + jsonWriter.createJson(response).toPrettyString()); + } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java index dfd9783ecef..1ad5f6828b7 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java @@ -30,7 +30,7 @@ import java.util.stream.Collectors; public abstract class StateRestApiTest { - private ClusterControllerMock books; + ClusterControllerMock books; ClusterControllerMock music; StateRestAPI restAPI; JsonWriter jsonWriter = new JsonWriter(); diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitMetrics.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitMetrics.java index f9876870873..f2c22b2dac5 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitMetrics.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/UnitMetrics.java @@ -5,6 +5,6 @@ import java.util.Map; public interface UnitMetrics { - Map<String, Long> getMetricMap(); + Map<String, Number> getMetricMap(); } diff --git a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java index a73e20b8755..9c39186855e 100644 --- a/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java +++ b/clustercontroller-utils/src/test/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/DummyStateApi.java @@ -139,8 +139,8 @@ public class DummyStateApi implements StateRestAPI { public UnitMetrics getMetrics() { return new UnitMetrics() { @Override - public Map<String, Long> getMetricMap() { - Map<String, Long> m = new LinkedHashMap<>(); + public Map<String, Number> getMetricMap() { + Map<String, Number> m = new LinkedHashMap<>(); m.put("doc-count", (long) node.docCount); return m; } diff --git a/config-application-package/src/main/java/com/yahoo/config/application/ConfigDefinitionDir.java b/config-application-package/src/main/java/com/yahoo/config/application/ConfigDefinitionDir.java index d4b257f0ba9..1329befbc9d 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/ConfigDefinitionDir.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/ConfigDefinitionDir.java @@ -11,7 +11,6 @@ import java.util.List; * but they cannot conflict with the existing ones. * * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigDefinitionDir { private final File defDir; diff --git a/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java index bc48e7dd814..ac365fa5a3e 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java @@ -22,7 +22,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; * Handles preprocess:include statements and returns a Document which has all the include statements resolved * * @author hmusum - * @since 5.22 */ class IncludeProcessor implements PreProcessor { diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java index bb456d95326..5f2046b1450 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java @@ -46,6 +46,7 @@ class OverrideProcessor implements PreProcessor { private final Tags tags; private static final String ID_ATTRIBUTE = "id"; + private static final String IDREF_ATTRIBUTE = "idref"; private static final String INSTANCE_ATTRIBUTE = "instance"; private static final String ENVIRONMENT_ATTRIBUTE = "environment"; private static final String REGION_ATTRIBUTE = "region"; @@ -200,7 +201,7 @@ class OverrideProcessor implements PreProcessor { /** Find the most specific element and remove all others. */ private void retainMostSpecific(Element parent, List<Element> children, Context context) { - // Keep track of elements with highest number of matches (might be more than one element with same tag, need a list) + // Keep track of elements with the highest number of matches (might be more than one element with same tag, need a list) List<Element> bestMatches = new ArrayList<>(); int bestMatch = 0; for (Element child : children) { @@ -307,42 +308,43 @@ class OverrideProcessor implements PreProcessor { private Set<InstanceName> getInstances(Element element) { String instance = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, INSTANCE_ATTRIBUTE); - if (instance == null || instance.isEmpty()) return Set.of(); + if (instance.isEmpty()) return Set.of(); return Arrays.stream(instance.split(" ")).map(InstanceName::from).collect(Collectors.toSet()); } private Set<Environment> getEnvironments(Element element) { String env = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, ENVIRONMENT_ATTRIBUTE); - if (env == null || env.isEmpty()) return Set.of(); + if (env.isEmpty()) return Set.of(); return Arrays.stream(env.split(" ")).map(Environment::from).collect(Collectors.toSet()); } private Set<RegionName> getRegions(Element element) { String reg = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, REGION_ATTRIBUTE); - if (reg == null || reg.isEmpty()) return Set.of(); + if (reg.isEmpty()) return Set.of(); return Arrays.stream(reg.split(" ")).map(RegionName::from).collect(Collectors.toSet()); } private Set<CloudName> getClouds(Element element) { String reg = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, CLOUD_ATTRIBUTE); - if (reg == null || reg.isEmpty()) return Set.of(); + if (reg.isEmpty()) return Set.of(); return Arrays.stream(reg.split(" ")).map(CloudName::from).collect(Collectors.toSet()); } private Tags getTags(Element element) { String env = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, TAGS_ATTRIBUTE); - if (env == null || env.isEmpty()) return Tags.empty(); + if (env.isEmpty()) return Tags.empty(); return Tags.fromString(env); } private Map<String, List<Element>> elementsByTagNameAndId(List<Element> children) { Map<String, List<Element>> elementsByTagName = new LinkedHashMap<>(); - // Index by tag name + // Index by tag name and optionally add "id" or "idref" to key if they are set for (Element child : children) { String key = child.getTagName(); - if (child.hasAttribute(ID_ATTRIBUTE)) { + if (child.hasAttribute(ID_ATTRIBUTE)) key += child.getAttribute(ID_ATTRIBUTE); - } + if (child.hasAttribute(IDREF_ATTRIBUTE)) + key += child.getAttribute(IDREF_ATTRIBUTE); if ( ! elementsByTagName.containsKey(key)) { elementsByTagName.put(key, new ArrayList<>()); } @@ -382,7 +384,7 @@ class OverrideProcessor implements PreProcessor { } /** - * Represents environment and region in a given context. + * Represents environments, regions, instances, clouds and tags in a given context. */ private static final class Context { diff --git a/config-application-package/src/main/java/com/yahoo/config/application/ValidationProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/ValidationProcessor.java index b02ccc711c0..4dc40df61f4 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/ValidationProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/ValidationProcessor.java @@ -11,8 +11,8 @@ public class ValidationProcessor implements PreProcessor { @Override public Document process(Document input) throws IOException, TransformerException { - NodeList includeitems = input.getElementsByTagNameNS("http://www.w3.org/2001/XInclude", "*"); - if (includeitems.getLength() > 0) + NodeList includeItems = input.getElementsByTagNameNS("http://www.w3.org/2001/XInclude", "*"); + if (includeItems.getLength() > 0) throw new UnsupportedOperationException("XInclude not supported, use preprocess:include instead"); return input; } diff --git a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java index e5e36615b09..c2b0770ab06 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java @@ -2,6 +2,7 @@ package com.yahoo.config.application; import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; @@ -12,6 +13,8 @@ import org.w3c.dom.Document; import javax.xml.transform.TransformerException; import java.io.StringReader; +import static com.yahoo.config.provision.Tags.empty; + /** * @author Ulf Lilleengen */ @@ -366,14 +369,72 @@ public class OverrideProcessorTest { assertOverride(input, Environment.dev, RegionName.defaultName(), expected); } + /** + * Tests that searchers referred to with idref are overridden per cloud + * and that searchers not referred to with idref are not overridden. + */ + @Test + public void testSearchersReferredWithIdRefPerCloud() throws TransformerException { + String input = + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="?" version="1.0"> + <container id="stateless" version="1.0"> + <search> + <searcher id="AwsSearcher" class="ai.vespa.AwsSearcher" bundle="foo"/> + <searcher id="GcpSearcher" class="ai.vespa.GcpSearcher" bundle="foo"/> + <searcher id="OtherSearcher" class="ai.vespa.OtherSearcher" bundle="foo"/> + <chain id="default" inherits="vespa"> + <searcher idref="AwsSearcher" deploy:cloud="aws"/> + <searcher idref="GcpSearcher" deploy:cloud="gcp"/> + <searcher idref="OtherSearcher"/> + </chain> + </search> + </container> + "</services>"""; + + String expected = + """ + <?xml version="1.0" encoding="UTF-8" standalone="no"?> + <services xmlns:deploy="vespa" xmlns:preprocess="?" version="1.0"> + <container id="stateless" version="1.0"> + <search> + <searcher id="AwsSearcher" class="ai.vespa.AwsSearcher" bundle="foo"/> + <searcher id="GcpSearcher" class="ai.vespa.GcpSearcher" bundle="foo"/> + <searcher id="OtherSearcher" class="ai.vespa.OtherSearcher" bundle="foo"/> + <chain id="default" inherits="vespa"> + <searcher idref="%s"/> + <searcher idref="OtherSearcher"/> + </chain> + </search> + </container> + "</services>"""; + + assertOverride(input, "aws", expected.formatted("AwsSearcher")); + assertOverride(input, "gcp", expected.formatted("GcpSearcher")); + } + private void assertOverride(Environment environment, RegionName region, String expected) throws TransformerException { assertOverride(input, environment, region, expected); } - private void assertOverride(String input, Environment environment, RegionName region, String expected) throws TransformerException { - Document inputDoc = Xml.getDocument(new StringReader(input)); - Document newDoc = new OverrideProcessor(InstanceName.from("default"), environment, region, Cloud.defaultCloud().name(), Tags.empty()).process(inputDoc); - TestBase.assertDocument(expected, newDoc); + private void assertOverride(String input, Environment environment, RegionName region, String expected) { + assertOverride(input, environment, region, Cloud.defaultCloud().name(), expected); + } + + private void assertOverride(String input, String cloudName, String expected) { + assertOverride(input, Environment.defaultEnvironment(), RegionName.defaultName(), CloudName.from(cloudName), expected); + } + + private void assertOverride(String input, Environment environment, RegionName region, CloudName cloudName, String expected) { + var inputDoc = Xml.getDocument(new StringReader(input)); + try { + var newDoc = new OverrideProcessor(InstanceName.from("default"), environment, region, cloudName, Tags.empty()) + .process(inputDoc); + TestBase.assertDocument(expected, newDoc); + } catch (TransformerException e) { + throw new RuntimeException(e); + } } } diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index f324ceef5ab..2461fc64172 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -233,10 +233,7 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP } } - // add cluster type? - // add cluster name? - public record StatePortInfo(String hostName, int portNumber, - String serviceName, String serviceType) + public record StatePortInfo(String hostName, int portNumber, Service service) {} public List<StatePortInfo> getStatePorts() { @@ -244,8 +241,6 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP for (HostResource modelHost : hostSystem().getHosts()) { String hostName = modelHost.getHostname(); for (Service modelService : modelHost.getServices()) { - String serviceName = modelService.getServiceName(); - String serviceType = modelService.getServiceType(); PortsMeta portsMeta = modelService.getPortsMeta(); for (int i = 0; i < portsMeta.getNumPorts(); i++) { int portNumber = modelService.getRelativePort(i); @@ -256,7 +251,7 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP if (tag.equals("http")) isHttp = true; } if (hasState && isHttp) { - result.add(new StatePortInfo(hostName, portNumber, serviceName, serviceType)); + result.add(new StatePortInfo(hostName, portNumber, modelService)); } } } diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 82ed45028b3..60674b5487c 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -141,6 +141,8 @@ public class RankProfile implements Cloneable { private Boolean strict; + private Boolean useSignificanceModel; + private final ApplicationPackage applicationPackage; private final DeployLogger deployLogger; @@ -216,6 +218,16 @@ public class RankProfile implements Cloneable { this.strict = strict; } + public void setUseSignificanceModel(Boolean useSignificanceModel) { + this.useSignificanceModel = useSignificanceModel; + } + + public boolean useSignificanceModel() { + if (useSignificanceModel != null) return useSignificanceModel; + return uniquelyInherited(p -> p.useSignificanceModel(), "use-model") + .orElse(false); // Disabled by default + } + /** * Adds a profile to those inherited by this. * The profile must belong to this schema (directly or by inheritance). diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index f996b2624db..b91404be2dd 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -183,10 +183,12 @@ public final class SchemaInfo extends Derived { private void addRankProfilesConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { for (RankProfileInfo rankProfile : rankProfiles().values()) { - var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileConfig.name(rankProfile.name()); - rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); - rankProfileConfig.hasRankFeatures(rankProfile.hasRankFeatures()); + var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder() + .name(rankProfile.name()) + .hasSummaryFeatures(rankProfile.hasSummaryFeatures()) + .hasRankFeatures(rankProfile.hasRankFeatures()) + .significance(new SchemaInfoConfig.Schema.Rankprofile.Significance.Builder() + .useModel(rankProfile.useSignificanceModel())); for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); @@ -226,6 +228,7 @@ public final class SchemaInfo extends Derived { private final String name; private final boolean hasSummaryFeatures; private final boolean hasRankFeatures; + private final boolean useSignificanceModel; private final Map<Reference, RankProfile.Input> inputs; public RankProfileInfo(RankProfile profile) { @@ -233,11 +236,13 @@ public final class SchemaInfo extends Derived { this.hasSummaryFeatures = ! profile.getSummaryFeatures().isEmpty(); this.hasRankFeatures = ! profile.getRankFeatures().isEmpty(); this.inputs = profile.inputs(); + useSignificanceModel = profile.useSignificanceModel(); } public String name() { return name; } public boolean hasSummaryFeatures() { return hasSummaryFeatures; } public boolean hasRankFeatures() { return hasRankFeatures; } + public boolean useSignificanceModel() { return useSignificanceModel; } public Map<Reference, RankProfile.Input> inputs() { return inputs; } } diff --git a/config-model/src/main/java/com/yahoo/schema/document/Matching.java b/config-model/src/main/java/com/yahoo/schema/document/Matching.java index 9d68553fa80..33256fa8586 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Matching.java @@ -33,6 +33,8 @@ public class Matching implements Cloneable, Serializable { private Integer maxLength; /** Maximum number of occurrences for each term */ private Integer maxTermOccurrences; + /** Maximum number of characters in a token. */ + private Integer maxTokenLength; private String exactMatchTerminator = null; @@ -61,6 +63,8 @@ public class Matching implements Cloneable, Serializable { public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; } public Integer maxTermOccurrences() { return maxTermOccurrences; } public Matching maxTermOccurrences(int maxTermOccurrences) { this.maxTermOccurrences = maxTermOccurrences; return this; } + public Integer maxTokenLength() { return maxTokenLength; } + public Matching maxTokenLength(int maxTokenLength) { this.maxTokenLength = maxTokenLength; return this; } public boolean isTypeUserSet() { return typeUserSet; } public MatchAlgorithm getAlgorithm() { return algorithm; } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 7659a1e6562..173eebe2a94 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -44,6 +44,7 @@ public class ConvertParsedFields { parsed.getGramSize().ifPresent(gramSize -> field.getMatching().setGramSize(gramSize)); parsed.getMaxLength().ifPresent(maxLength -> field.getMatching().maxLength(maxLength)); parsed.getMaxTermOccurrences().ifPresent(maxTermOccurrences -> field.getMatching().maxTermOccurrences(maxTermOccurrences)); + parsed.getMaxTokenLength().ifPresent(maxTokenLength -> field.getMatching().maxTokenLength(maxTokenLength)); parsed.getMatchAlgorithm().ifPresent (matchingAlgorithm -> field.setMatchingAlgorithm(matchingAlgorithm)); parsed.getExactTerminator().ifPresent diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java index 5ccbb7b19a4..77a10862f9c 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java @@ -39,6 +39,7 @@ public class ConvertParsedRanking { profile.inherit(name); parsed.isStrict().ifPresent(value -> profile.setStrict(value)); + parsed.isUseSignificanceModel().ifPresent(value -> profile.setUseSignificanceModel(value)); for (var constant : parsed.getConstants().values()) profile.add(constant); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java index 789f7023aed..063962bf0c4 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java @@ -133,9 +133,9 @@ public class IntermediateCollection { var parser = new SchemaParser(stream, deployLogger, modelProperties); try { parser.rankProfile(schema); - } catch (ParseException pe) { + } catch (ParseException | TokenMgrException e) { throw new ParseException("Failed parsing rank-profile from '" + reader.getName() + "': " + - stream.formatException(Exceptions.toMessageString(pe))); + stream.formatException(Exceptions.toMessageString(e))); } } catch (java.io.IOException ex) { throw new IllegalArgumentException("Failed reading from '" + reader.getName() + "': " + ex.getMessage()); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java index c7d1a215ce3..bac2c894283 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java @@ -23,6 +23,7 @@ public class ParsedMatchSettings { private Integer gramSize = null; private Integer maxLength = null; private Integer maxTermOccurrences = null; + private Integer maxTokenLength = null; Optional<MatchType> getMatchType() { return Optional.ofNullable(matchType); } Optional<Case> getMatchCase() { return Optional.ofNullable(matchCase); } @@ -31,6 +32,7 @@ public class ParsedMatchSettings { Optional<Integer> getGramSize() { return Optional.ofNullable(gramSize); } Optional<Integer> getMaxLength() { return Optional.ofNullable(maxLength); } Optional<Integer> getMaxTermOccurrences() { return Optional.ofNullable(maxTermOccurrences); } + Optional<Integer> getMaxTokenLength() { return Optional.ofNullable(maxTokenLength); } // TODO - consider allowing each set only once: void setType(MatchType value) { this.matchType = value; } @@ -40,5 +42,6 @@ public class ParsedMatchSettings { void setGramSize(int value) { this.gramSize = value; } void setMaxLength(int value) { this.maxLength = value; } void setMaxTermOccurrences(int value) { this.maxTermOccurrences = value; } + void setMaxTokenLength(int value) { this.maxTokenLength = value; } } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java index fbbb0c7fe83..93319e82076 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java @@ -44,6 +44,7 @@ class ParsedRankProfile extends ParsedBlock { private String inheritedMatchFeatures = null; private String secondPhaseExpression = null; private Boolean strict = null; + private Boolean useSignificanceModel = null; private final List<MutateOperation> mutateOperations = new ArrayList<>(); private final List<String> inherited = new ArrayList<>(); private final Map<String, Boolean> fieldsRankFilter = new LinkedHashMap<>(); @@ -96,6 +97,8 @@ class ParsedRankProfile extends ParsedBlock { Optional<String> getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } Optional<Boolean> isStrict() { return Optional.ofNullable(this.strict); } + Optional<Boolean> isUseSignificanceModel() { return Optional.ofNullable(this.useSignificanceModel); } + void addSummaryFeatures(FeatureList features) { this.summaryFeatures.add(features); } void addMatchFeatures(FeatureList features) { this.matchFeatures.add(features); } void addRankFeatures(FeatureList features) { this.rankFeatures.add(features); } @@ -218,6 +221,10 @@ class ParsedRankProfile extends ParsedBlock { this.strict = strict; } + void setUseSignificanceModel(boolean useSignificanceModel) { + verifyThat(this.useSignificanceModel == null, "already has use-model"); + this.useSignificanceModel = useSignificanceModel; + } void setTermwiseLimit(double limit) { verifyThat(termwiseLimit == null, "already has termwise-limit"); this.termwiseLimit = limit; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java index 056c37a9830..4313ceb4be1 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; import com.yahoo.vespa.model.container.search.QueryProfiles; /** @@ -75,7 +76,11 @@ public class ExactMatch extends Processor { } ScriptExpression script = field.getIndexingScript(); if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) { - field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); + var maxTokenLength = field.getMatching().maxTokenLength(); + if (maxTokenLength == null) { + maxTokenLength = AnnotatorConfig.getDefaultMaxTokenLength(); + } + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema, maxTokenLength).convert(field.getIndexingScript())); } } @@ -85,8 +90,12 @@ public class ExactMatch extends Processor { private static class MyProvider extends TypedTransformProvider { - MyProvider(Schema schema) { + private int maxTokenLength; + + MyProvider(Schema schema, int maxTokenLength) + { super(ExactExpression.class, schema); + this.maxTokenLength = maxTokenLength; } @Override @@ -96,7 +105,7 @@ public class ExactMatch extends Processor { @Override protected Expression newTransform(DataType fieldType) { - Expression exp = new ExactExpression(); + Expression exp = new ExactExpression(maxTokenLength); if (fieldType instanceof CollectionDataType) { exp = new ForEachExpression(exp); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java index e29f683761f..3f23cbc9b2d 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java @@ -64,12 +64,16 @@ public class TextMatch extends Processor { if (fieldMatching != null) { var maxLength = fieldMatching.maxLength(); if (maxLength != null) { - ret.setMaxTokenLength(maxLength); + ret.setMaxTokenizeLength(maxLength); } var maxTermOccurrences = fieldMatching.maxTermOccurrences(); if (maxTermOccurrences != null) { ret.setMaxTermOccurrences(maxTermOccurrences); } + var maxTokenLength = fieldMatching.maxTokenLength(); + if (maxTokenLength != null) { + ret.setMaxTokenLength(maxTokenLength); + } } return ret; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java index 8ccc8870419..3d4934ed841 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java @@ -29,6 +29,10 @@ public abstract class TypedTransformProvider extends ValueTransformProvider { protected final boolean requiresTransform(Expression exp) { if (exp instanceof OutputExpression) { String fieldName = ((OutputExpression)exp).getFieldName(); + if (fieldName == null) { + // Incomplete output expressions never require a transform. + return false; + } if (exp instanceof AttributeExpression) { Attribute attribute = schema.getAttribute(fieldName); if (attribute == null) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index d8149486b32..806f14da265 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -342,6 +342,7 @@ public abstract class AbstractService extends TreeConfigProducer<AnyConfigProduc return getServicePropertyString(key, null); } + @Override public String getServicePropertyString(String key, String defStr) { Object result = serviceProperties.get(key); return (result == null) ? defStr : result.toString(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java index fcd587622da..03b96b12c03 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java @@ -5,6 +5,7 @@ import com.yahoo.cloud.config.OpenTelemetryConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.PortAllocBridge; @@ -17,10 +18,12 @@ import java.util.Optional; public class OpenTelemetryCollector extends AbstractService implements OpenTelemetryConfig.Producer { private final Zone zone; + private final ApplicationId applicationId; public OpenTelemetryCollector(TreeConfigProducer<?> parent) { super(parent, "otelcol"); this.zone = null; + this.applicationId = null; setProp("clustertype", "admin"); setProp("clustername", "admin"); } @@ -28,6 +31,7 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem public OpenTelemetryCollector(TreeConfigProducer<?> parent, DeployState deployState) { super(parent, "otelcol"); this.zone = deployState.zone(); + this.applicationId = deployState.getProperties().applicationId(); setProp("clustertype", "admin"); setProp("clustername", "admin"); } @@ -50,7 +54,7 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem @Override public void getConfig(OpenTelemetryConfig.Builder builder) { - var generator = new OpenTelemetryConfigGenerator(zone); + var generator = new OpenTelemetryConfigGenerator(zone, applicationId); AnyConfigProducer pp = this; AnyConfigProducer p = pp.getParent(); while (p != null && p != pp) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java index 36eab6a04b3..3f7ca7b46a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java @@ -1,16 +1,23 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.otel; +import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.Service; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import static com.yahoo.vespa.defaults.Defaults.getDefaults; @@ -24,8 +31,12 @@ public class OpenTelemetryConfigGenerator { private final String cert_file; private final String key_file; private List<StatePortInfo> statePorts = new ArrayList<>(); + private final Zone zone; + private final ApplicationId applicationId; - OpenTelemetryConfigGenerator(Zone zone) { + OpenTelemetryConfigGenerator(Zone zone, ApplicationId applicationId) { + this.zone = zone; + this.applicationId = applicationId; boolean isCd = true; boolean isPublic = true; if (zone != null) { @@ -81,8 +92,15 @@ public class OpenTelemetryConfigGenerator { if (useTls) addTls(g); { g.writeFieldName("labels"); + var dimVals = serviceAttributes(statePort.service()); + // these will be tagged as dimension-name/label-value + // attributes on all metrics from this /state/v1 port g.writeStartObject(); - g.writeStringField("service_type", statePort.serviceType()); + for (var entry : dimVals.entrySet()) { + if (entry.getValue() != null) { + g.writeStringField(entry.getKey(), entry.getValue()); + } + } g.writeEndObject(); } g.writeEndObject(); @@ -123,6 +141,37 @@ public class OpenTelemetryConfigGenerator { } g.writeEndObject(); // file } + private void addProcessors(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("processors"); + g.writeStartObject(); + addResourceProcessor(g); + g.writeEndObject(); + } + private void addResourceProcessor(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("resource"); + g.writeStartObject(); + g.writeFieldName("attributes"); + g.writeStartArray(); + // common attributes for all metrics from all services; + // which application and which cloud/system/zone/environment + addAttributeInsert(g, PublicDimensions.ZONE, zoneAttr()); + addAttributeInsert(g, PublicDimensions.APPLICATION_ID, appIdAttr()); + addAttributeInsert(g, "system", systemAttr()); + addAttributeInsert(g, "tenantName", tenantAttr()); + addAttributeInsert(g, "applicationName", appNameAttr()); + addAttributeInsert(g, "instanceName", appInstanceAttr()); + addAttributeInsert(g, "cloud", cloudAttr()); + g.writeEndArray(); + g.writeEndObject(); + } + private void addAttributeInsert(JsonGenerator g, String key, String value) throws java.io.IOException { + if (value == null) return; + g.writeStartObject(); + g.writeStringField("key", key); + g.writeStringField("value", value); + g.writeStringField("action", "insert"); + g.writeEndObject(); + } private void addServiceBlock(JsonGenerator g) throws java.io.IOException { g.writeFieldName("service"); g.writeStartObject(); @@ -159,6 +208,7 @@ public class OpenTelemetryConfigGenerator { } g.writeFieldName("processors"); g.writeStartArray(); + g.writeString("resource"); g.writeEndArray(); { g.writeFieldName("exporters"); @@ -186,6 +236,7 @@ public class OpenTelemetryConfigGenerator { g.writeStartObject(); addReceivers(g); addExporters(g); + addProcessors(g); addServiceBlock(g); g.writeEndObject(); // root g.close(); @@ -203,4 +254,70 @@ public class OpenTelemetryConfigGenerator { List<String> referencedPaths() { return List.of(ca_file, cert_file, key_file); } + + private String zoneAttr() { + if (zone == null) return null; + return zone.environment().value() + "." + zone.region().value(); + } + private String appIdAttr() { + if (applicationId == null) return null; + return applicationId.toFullString(); + } + private String systemAttr() { + if (zone == null) return null; + return zone.system().value(); + } + private String tenantAttr() { + if (applicationId == null) return null; + return applicationId.tenant().value(); + } + private String appNameAttr() { + if (applicationId == null) return null; + return applicationId.application().value(); + } + private String appInstanceAttr() { + if (applicationId == null) return null; + return applicationId.instance().value(); + } + private String cloudAttr() { + if (zone == null) return null; + return zone.cloud().name().value(); + } + + private String getDeploymentCluster(ClusterSpec cluster) { + if (applicationId == null) return null; + if (zone == null) return null; + String appString = applicationId.toFullString(); + return String.join(".", appString, + zone.environment().value(), + zone.region().value(), + cluster.id().value()); + } + + private Map<String, String> serviceAttributes(Service svc) { + Map<String, String> dimvals = new LinkedHashMap<>(); + dimvals.put("instance", svc.getServiceName()); // should maybe be "local_service_name" ? + dimvals.put("instanceType", svc.getServiceType()); // maybe "local_service_type", or remove + String cName = svc.getServicePropertyString("clustername", null); + if (cName != null) { + // what about "clusterid" below, is it always the same? + dimvals.put("clustername", cName); + } + String cType = svc.getServicePropertyString("clustertype", null); + if (cType != null) { + dimvals.put("clustertype", cType); + } + var hostResource = svc.getHost(); + if (hostResource != null) { + hostResource.spec().membership().map(ClusterMembership::cluster).ifPresent(cluster -> { + dimvals.put(PublicDimensions.DEPLOYMENT_CLUSTER, getDeploymentCluster(cluster)); + // overrides value above + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cluster.type().name()); + // alternative to above + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_ID, cluster.id().value()); + cluster.group().ifPresent(group -> dimvals.put(PublicDimensions.GROUP_ID, group.toString())); + }); + } + return dimvals; + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java index 40c9a03b126..02a6b243054 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java @@ -132,7 +132,7 @@ public class ConstantTensorJsonValidator { private void consumeTopObject() throws IOException { for (var cur = parser.nextToken(); cur != JsonToken.END_OBJECT; cur = parser.nextToken()) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.getCurrentName(); + String fieldName = parser.currentName(); switch (fieldName) { case FIELD_TYPE -> consumeTypeField(); case FIELD_VALUES -> consumeValuesField(); @@ -189,7 +189,7 @@ public class ConstantTensorJsonValidator { } for (var cur = parser.nextToken(); cur != JsonToken.END_OBJECT; cur = parser.nextToken()) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - validateNumeric(parser.getCurrentName(), parser.nextToken()); + validateNumeric(parser.currentName(), parser.nextToken()); } } @@ -199,7 +199,7 @@ public class ConstantTensorJsonValidator { boolean seenValue = false; for (int i = 0; i < 2; i++) { assertNextTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.getCurrentName(); + String fieldName = parser.currentName(); switch (fieldName) { case FIELD_ADDRESS -> { validateTensorAddress(new HashSet<>(tensorDimensions.keySet())); @@ -228,13 +228,13 @@ public class ConstantTensorJsonValidator { // Iterate within the address key, value pairs while ((parser.nextToken() != JsonToken.END_OBJECT)) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - String dimensionName = parser.getCurrentName(); + String dimensionName = parser.currentName(); TensorType.Dimension dimension = tensorDimensions.get(dimensionName); if (dimension == null) { - throw new InvalidConstantTensorException(parser, String.format("Tensor dimension '%s' does not exist", parser.getCurrentName())); + throw new InvalidConstantTensorException(parser, String.format("Tensor dimension '%s' does not exist", dimensionName)); } if (!cellDimensions.contains(dimensionName)) { - throw new InvalidConstantTensorException(parser, String.format("Duplicate tensor dimension '%s'", parser.getCurrentName())); + throw new InvalidConstantTensorException(parser, String.format("Duplicate tensor dimension '%s'", dimensionName)); } cellDimensions.remove(dimensionName); validateLabel(dimension); @@ -300,7 +300,7 @@ public class ConstantTensorJsonValidator { } private void assertCurrentTokenIs(JsonToken wantedToken) { - assertTokenIs(parser.getCurrentToken(), wantedToken); + assertTokenIs(parser.currentToken(), wantedToken); } private void assertNextTokenIs(JsonToken wantedToken) throws IOException { @@ -316,11 +316,11 @@ public class ConstantTensorJsonValidator { static class InvalidConstantTensorException extends IllegalArgumentException { InvalidConstantTensorException(JsonParser parser, String message) { - super(message + " " + parser.getCurrentLocation().toString()); + super(message + " " + parser.currentLocation().toString()); } InvalidConstantTensorException(JsonParser parser, Exception base) { - super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base); + super("Failed to parse JSON stream " + parser.currentLocation().toString(), base); } InvalidConstantTensorException(IOException base) { @@ -412,7 +412,7 @@ public class ConstantTensorJsonValidator { boolean seenValues = false; for (int i = 0; i < 2; i++) { assertNextTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.getCurrentName(); + String fieldName = parser.currentName(); switch (fieldName) { case FIELD_ADDRESS -> { validateTensorAddress(new HashSet<>(mappedDims)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index ed0804f7420..7f624032627 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -19,6 +19,7 @@ import com.yahoo.vespa.model.application.validation.change.IndexingModeChangeVal import com.yahoo.vespa.model.application.validation.change.NodeResourceChangeValidator; import com.yahoo.vespa.model.application.validation.change.RedundancyIncreaseValidator; import com.yahoo.vespa.model.application.validation.change.ResourcesReductionValidator; +import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForLocalLLMValidator; import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForOnnxModelChangesValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; @@ -129,6 +130,7 @@ public class Validation { new CertificateRemovalChangeValidator().validate(execution); new RedundancyValidator().validate(execution); new RestartOnDeployForOnnxModelChangesValidator().validate(execution); + new RestartOnDeployForLocalLLMValidator().validate(execution); } public interface Context { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java new file mode 100644 index 00000000000..ccfc611c3dc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java @@ -0,0 +1,55 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; + +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +import static java.util.logging.Level.INFO; +import static java.util.stream.Collectors.toUnmodifiableSet; + +/** + * If using local LLMs, this validator will make sure that restartOnDeploy is set for + * configs for this cluster. + * + * @author lesters + */ +public class RestartOnDeployForLocalLLMValidator implements ChangeValidator { + + public static final String LOCAL_LLM_COMPONENT = ai.vespa.llm.clients.LocalLLM.class.getName(); + + private static final Logger log = Logger.getLogger(RestartOnDeployForLocalLLMValidator.class.getName()); + + @Override + public void validate(ChangeContext context) { + var previousClustersWithLocalLLM = findClustersWithLocalLLMs(context.previousModel()); + var nextClustersWithLocalLLM = findClustersWithLocalLLMs(context.model()); + + // Only restart services if we use a local LLM in both the next and previous generation + for (var clusterId : intersect(previousClustersWithLocalLLM, nextClustersWithLocalLLM)) { + String message = "Need to restart services in %s due to use of local LLM".formatted(clusterId); + context.require(new VespaRestartAction(clusterId, message)); + log.log(INFO, message); + } + } + + private Set<ClusterSpec.Id> findClustersWithLocalLLMs(VespaModel model) { + return model.getContainerClusters().values().stream() + .filter(cluster -> cluster.getAllComponents().stream() + .anyMatch(component -> component.getClassId().getName().equals(LOCAL_LLM_COMPONENT))) + .map(ApplicationContainerCluster::id) + .collect(toUnmodifiableSet()); + } + + private Set<ClusterSpec.Id> intersect(Set<ClusterSpec.Id> a, Set<ClusterSpec.Id> b) { + Set<ClusterSpec.Id> result = new HashSet<>(a); + result.retainAll(b); + return result; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java index c210c2621a6..693eebd75a8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java @@ -33,17 +33,15 @@ public class SignificanceModelRegistry extends SimpleComponent implements Signif if (spec != null) { for (Element modelElement : XML.getChildren(spec, "model")) { - addConfig( - modelElement.getAttribute("language"), - Model.fromXml(deployState, modelElement, Set.of(SIGNIFICANCE_MODEL)).modelReference()); + addConfig(Model.fromXml(deployState, modelElement, Set.of(SIGNIFICANCE_MODEL)).modelReference()); } } } - public void addConfig(String language, ModelReference path) { + public void addConfig(ModelReference path) { configList.add( - new SignificanceModelConfig(language, path) + new SignificanceModelConfig(path) ); } @@ -53,19 +51,16 @@ public class SignificanceModelRegistry extends SimpleComponent implements Signif builder.model( configList.stream() .map(config -> new SignificanceConfig.Model.Builder() - .language(config.language) .path(config.path) ).toList() ); } - class SignificanceModelConfig { - private final String language; + static class SignificanceModelConfig { private final ModelReference path; - public SignificanceModelConfig(String language, ModelReference path) { - this.language = language; + public SignificanceModelConfig(ModelReference path) { this.path = path; } diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 255cc3cde70..1365c133932 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -183,11 +183,14 @@ TOKEN : | < GRAM_SIZE: "gram-size" > | < MAX_LENGTH: "max-length" > | < MAX_OCCURRENCES: "max-occurrences" > +| < MAX_TOKEN_LENGTH: "max-token-length" > | < PREFIX: "prefix" > | < SUBSTRING: "substring" > | < SUFFIX: "suffix" > | < CONSTANT: "constant"> | < ONNX_MODEL: "onnx-model"> +| < SIGNIFICANCE: "significance"> +| < USE_MODEL: "use-model"> | < INTRAOP_THREADS: "intraop-threads"> | < INTEROP_THREADS: "interop-threads"> | < GPU_DEVICE: "gpu-device"> @@ -1366,7 +1369,8 @@ void matchType(ParsedMatchSettings matchInfo) : { } */ void matchItem(ParsedMatchSettings matchInfo) : { } { - ( matchType(matchInfo) | exactTerminator(matchInfo) | gramSize(matchInfo) | matchSize(matchInfo) | maxTermOccurrences(matchInfo)) + ( matchType(matchInfo) | exactTerminator(matchInfo) | gramSize(matchInfo) | matchSize(matchInfo) | + maxTermOccurrences(matchInfo) | maxTokenLength(matchInfo) ) } void exactTerminator(ParsedMatchSettings matchInfo) : @@ -1411,6 +1415,16 @@ void maxTermOccurrences(ParsedMatchSettings matchInfo) : } } +void maxTokenLength(ParsedMatchSettings matchInfo) : +{ + int maxTokenLength; +} +{ + <MAX_TOKEN_LENGTH> <COLON> maxTokenLength = integer() { + matchInfo.setMaxTokenLength(maxTokenLength); + } +} + /** * Consumes a rank statement of a field element. * @@ -1761,7 +1775,8 @@ void rankProfileItem(ParsedSchema schema, ParsedRankProfile profile) : { } | matchFeatures(profile) | summaryFeatures(profile) | onnxModelInProfile(profile) - | strict(profile) ) + | strict(profile) + | significance(profile)) } /** @@ -2115,6 +2130,22 @@ void strict(ParsedRankProfile profile) : ) } +void significance(ParsedRankProfile profile) : +{} +{ + <SIGNIFICANCE> lbrace() (significanceItem(profile) (<NL>)*)* <RBRACE> + {} +} + +void significanceItem(ParsedRankProfile profile) : +{} +{ + <USE_MODEL> <COLON> ( + ( <TRUE> { profile.setUseSignificanceModel(true); } ) | + ( <FALSE> { profile.setUseSignificanceModel(false); } ) + ) +} + /** * Consumes a match-features block of a rank profile. * @@ -2710,6 +2741,7 @@ String identifierWithDash() : | <TARGET_HITS_MAX_ADJUSTMENT_FACTOR> | <TERMWISE_LIMIT> | <UPPER_BOUND> + | <USE_MODEL> ) { return token.image; } } @@ -2812,6 +2844,7 @@ String identifier() : { } | <STEMMING> | <STRENGTH> | <STRICT> + | <SIGNIFICANCE> | <STRING> | <STRUCT> | <SUBSTRING> diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 08092f10020..c79a7b38d09 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -138,7 +138,7 @@ Threadpool = element threadpool { } Significance = element significance { - element model { attribute language { xsd:string } & ModelReference }* + element model { ModelReference }* } Clients = element clients { diff --git a/config-model/src/test/cfg/significance/services.xml b/config-model/src/test/cfg/significance/services.xml index 6991f5498fb..ffdb73bfc2e 100644 --- a/config-model/src/test/cfg/significance/services.xml +++ b/config-model/src/test/cfg/significance/services.xml @@ -8,9 +8,9 @@ <container version="1.0"> <search> <significance> - <model language="en" model-id="idf-wiki-english" path="models/idf-english-wiki.json.zst"/> - <model language="no" path="models/idf-norwegian-wiki.json.zst" /> - <model language="ru" url="https://some/uri/blob.json" /> + <model model-id="idf-wiki-english" path="models/idf-english-wiki.json.zst"/> + <model path="models/idf-norwegian-wiki.json.zst" /> + <model url="https://some/uri/blob.json" /> </significance> </search> </container> diff --git a/config-model/src/test/derived/advanced/ilscripts.cfg b/config-model/src/test/derived/advanced/ilscripts.cfg index 51a49502b64..d633cd97f0c 100644 --- a/config-model/src/test/derived/advanced/ilscripts.cfg +++ b/config-model/src/test/derived/advanced/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "advanced" ilscript[].docfield[] "debug_src" diff --git a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg index 767c3af3c19..53dc789fbb7 100644 --- a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsimplicitstruct" diff --git a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg index d8e6c882b80..b0a69c5408a 100644 --- a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsinheritance" diff --git a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg index ae4ea621583..5ec1f839429 100644 --- a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsinheritance2" diff --git a/config-model/src/test/derived/annotationsreference/ilscripts.cfg b/config-model/src/test/derived/annotationsreference/ilscripts.cfg index 812f5e44545..eaa20043be8 100644 --- a/config-model/src/test/derived/annotationsreference/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsreference/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsreference" diff --git a/config-model/src/test/derived/annotationssimple/ilscripts.cfg b/config-model/src/test/derived/annotationssimple/ilscripts.cfg index 9d0962df5be..af179221eb4 100644 --- a/config-model/src/test/derived/annotationssimple/ilscripts.cfg +++ b/config-model/src/test/derived/annotationssimple/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationssimple" diff --git a/config-model/src/test/derived/arrays/ilscripts.cfg b/config-model/src/test/derived/arrays/ilscripts.cfg index 98cff642d9e..3f2dae48552 100644 --- a/config-model/src/test/derived/arrays/ilscripts.cfg +++ b/config-model/src/test/derived/arrays/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "arrays" ilscript[].docfield[] "tags" diff --git a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg index dec054b33f0..5a3784f7cb9 100644 --- a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg +++ b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "prefetch" ilscript[].docfield[] "singlebyte" diff --git a/config-model/src/test/derived/attributes/ilscripts.cfg b/config-model/src/test/derived/attributes/ilscripts.cfg index 6d3ef2799d9..58279759e5f 100644 --- a/config-model/src/test/derived/attributes/ilscripts.cfg +++ b/config-model/src/test/derived/attributes/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "attributes" ilscript[].docfield[] "a1" diff --git a/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg index c20c321ebcf..0b925da4778 100644 --- a/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg +++ b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "test" ilscript[].docfield[] "str_1" diff --git a/config-model/src/test/derived/complex/ilscripts.cfg b/config-model/src/test/derived/complex/ilscripts.cfg index 4405d2fda40..7d025e15703 100644 --- a/config-model/src/test/derived/complex/ilscripts.cfg +++ b/config-model/src/test/derived/complex/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "complex" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/emptydefault/ilscripts.cfg b/config-model/src/test/derived/emptydefault/ilscripts.cfg index e4242153bce..bbb8e5c556c 100644 --- a/config-model/src/test/derived/emptydefault/ilscripts.cfg +++ b/config-model/src/test/derived/emptydefault/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "emptydefault" ilscript[].docfield[] "one" diff --git a/config-model/src/test/derived/exactmatch/ilscripts.cfg b/config-model/src/test/derived/exactmatch/ilscripts.cfg index 21dfbd1371b..1d1bd6d5e8a 100644 --- a/config-model/src/test/derived/exactmatch/ilscripts.cfg +++ b/config-model/src/test/derived/exactmatch/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "exactmatch" ilscript[].docfield[] "tag" diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg index e48f116f468..c811b93c3df 100644 --- a/config-model/src/test/derived/hnsw_index/ilscripts.cfg +++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "test" ilscript[].docfield[] "t1" diff --git a/config-model/src/test/derived/id/ilscripts.cfg b/config-model/src/test/derived/id/ilscripts.cfg index d3ab29f6cd8..121e305059e 100644 --- a/config-model/src/test/derived/id/ilscripts.cfg +++ b/config-model/src/test/derived/id/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "id" ilscript[].docfield[] "uri" diff --git a/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg b/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg index f820ad9720b..5a474f62e07 100644 --- a/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg +++ b/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg @@ -53,6 +53,8 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false diff --git a/config-model/src/test/derived/indexswitches/ilscripts.cfg b/config-model/src/test/derived/indexswitches/ilscripts.cfg index 472c1f95cb0..454f675c0a2 100644 --- a/config-model/src/test/derived/indexswitches/ilscripts.cfg +++ b/config-model/src/test/derived/indexswitches/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "indexswitches" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/inheritance/ilscripts.cfg b/config-model/src/test/derived/inheritance/ilscripts.cfg index d4c804773f0..c966f32a502 100644 --- a/config-model/src/test/derived/inheritance/ilscripts.cfg +++ b/config-model/src/test/derived/inheritance/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "child" ilscript[].docfield[] "onlygrandparent" diff --git a/config-model/src/test/derived/language/ilscripts.cfg b/config-model/src/test/derived/language/ilscripts.cfg index 1860f180839..d0abc08f1e0 100644 --- a/config-model/src/test/derived/language/ilscripts.cfg +++ b/config-model/src/test/derived/language/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "language" ilscript[].docfield[] "language" diff --git a/config-model/src/test/derived/lowercase/ilscripts.cfg b/config-model/src/test/derived/lowercase/ilscripts.cfg index 8ba4bfa3349..49515e50df4 100644 --- a/config-model/src/test/derived/lowercase/ilscripts.cfg +++ b/config-model/src/test/derived/lowercase/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "lowercase" ilscript[].docfield[] "single_field_source" diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg index 0cdf921de25..4a6de4154f8 100644 --- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg +++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "multiplesummaries" ilscript[].docfield[] "a" diff --git a/config-model/src/test/derived/music/ilscripts.cfg b/config-model/src/test/derived/music/ilscripts.cfg index f90cdb15baa..f79e8824b69 100644 --- a/config-model/src/test/derived/music/ilscripts.cfg +++ b/config-model/src/test/derived/music/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "music" ilscript[].docfield[] "bgndata" diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg index 728856abbf2..8f59c21e97f 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg @@ -156,6 +156,7 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(W_0)" schema[].rankprofile[].input[].type "tensor(hidden[9],x[9])" schema[].rankprofile[].input[].name "query(b_0)" @@ -173,9 +174,11 @@ schema[].rankprofile[].input[].type "tensor()" schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "defaultRankProfile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(W_0)" schema[].rankprofile[].input[].type "tensor(hidden[9],x[9])" schema[].rankprofile[].input[].name "query(b_0)" @@ -193,6 +196,7 @@ schema[].rankprofile[].input[].type "tensor()" schema[].rankprofile[].name "neuralNetworkProfile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(W_0)" schema[].rankprofile[].input[].type "tensor(hidden[9],x[9])" schema[].rankprofile[].input[].name "query(b_0)" diff --git a/config-model/src/test/derived/newrank/ilscripts.cfg b/config-model/src/test/derived/newrank/ilscripts.cfg index b02e09a0496..487d2fca902 100644 --- a/config-model/src/test/derived/newrank/ilscripts.cfg +++ b/config-model/src/test/derived/newrank/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "newrank" ilscript[].docfield[] "bgndata" diff --git a/config-model/src/test/derived/orderilscripts/ilscripts.cfg b/config-model/src/test/derived/orderilscripts/ilscripts.cfg index 0ed1589af0a..4918e23efc6 100644 --- a/config-model/src/test/derived/orderilscripts/ilscripts.cfg +++ b/config-model/src/test/derived/orderilscripts/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "orderilscripts" ilscript[].docfield[] "foo" diff --git a/config-model/src/test/derived/position_array/ilscripts.cfg b/config-model/src/test/derived/position_array/ilscripts.cfg index ecafbc4a025..3f7611b25d8 100644 --- a/config-model/src/test/derived/position_array/ilscripts.cfg +++ b/config-model/src/test/derived/position_array/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_array" ilscript[].docfield[] "pos" diff --git a/config-model/src/test/derived/position_attribute/ilscripts.cfg b/config-model/src/test/derived/position_attribute/ilscripts.cfg index d2fc8503ce5..fbd1a293418 100644 --- a/config-model/src/test/derived/position_attribute/ilscripts.cfg +++ b/config-model/src/test/derived/position_attribute/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_attribute" ilscript[].docfield[] "pos" diff --git a/config-model/src/test/derived/position_extra/ilscripts.cfg b/config-model/src/test/derived/position_extra/ilscripts.cfg index a86dcec92ec..4645798723c 100644 --- a/config-model/src/test/derived/position_extra/ilscripts.cfg +++ b/config-model/src/test/derived/position_extra/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_extra" ilscript[].docfield[] "pos_str" diff --git a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg index 40c7843a0a4..2d1904cf9d8 100644 --- a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg +++ b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "prefixexactattribute" ilscript[].docfield[] "indexfield0" diff --git a/config-model/src/test/derived/rankingexpression/schema-info.cfg b/config-model/src/test/derived/rankingexpression/schema-info.cfg index 5bf01f10ede..f78eb7de310 100644 --- a/config-model/src/test/derived/rankingexpression/schema-info.cfg +++ b/config-model/src/test/derived/rankingexpression/schema-info.cfg @@ -148,96 +148,125 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures true +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "static" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "overflow" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "duplicates" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "whitespace1" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "whitespace2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros2" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros3" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros3-inherited" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-inherited" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-inherited2" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-inherited3" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros-inherited" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros-inherited2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros-inherited-two-levels" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withmf" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withboolean" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withglobalphase" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "layered" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(v)" schema[].rankprofile[].input[].type "tensor(v[3])" schema[].rankprofile[].name "withtfl" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(v)" schema[].rankprofile[].input[].type "tensor(v[3])" schema[].rankprofile[].name "withtfl2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(v)" schema[].rankprofile[].input[].type "tensor(v[3])" schema[].rankprofile[].name "withnorm" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withfusion" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "function-with-arg-as-summary-feature" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "function-with-arg-in-global-phase" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withstringcompare" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(myquerystring)" schema[].rankprofile[].input[].type "string" schema[].rankprofile[].input[].name "query(mybadlong)" diff --git a/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg b/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg index 377c10d3293..68892737e63 100644 --- a/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg +++ b/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg @@ -18,24 +18,32 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema0" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema1" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema3" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "outside_schema1" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "outside_schema2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false diff --git a/config-model/src/test/derived/rankprofilemodularity2/invalid_comment.profile b/config-model/src/test/derived/rankprofilemodularity2/invalid_comment.profile new file mode 100644 index 00000000000..40e77c8c6be --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity2/invalid_comment.profile @@ -0,0 +1,8 @@ +rank-profile outside_schema1 { + + // Comment with wrong comment character + function foo() { + expression: now + } + +} diff --git a/config-model/src/test/derived/ranktypes/ilscripts.cfg b/config-model/src/test/derived/ranktypes/ilscripts.cfg index adcd2f70c70..22526d1aa23 100644 --- a/config-model/src/test/derived/ranktypes/ilscripts.cfg +++ b/config-model/src/test/derived/ranktypes/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "ranktypes" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/schemainheritance/ilscripts.cfg b/config-model/src/test/derived/schemainheritance/ilscripts.cfg index f7324920fe7..b1ba947f1dc 100644 --- a/config-model/src/test/derived/schemainheritance/ilscripts.cfg +++ b/config-model/src/test/derived/schemainheritance/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "child" ilscript[].docfield[] "pf1" diff --git a/config-model/src/test/derived/schemainheritance/schema-info.cfg b/config-model/src/test/derived/schemainheritance/schema-info.cfg index 9fe71780c7a..466e66ad0bb 100644 --- a/config-model/src/test/derived/schemainheritance/schema-info.cfg +++ b/config-model/src/test/derived/schemainheritance/schema-info.cfg @@ -116,12 +116,16 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "child_profile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "parent_profile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false diff --git a/config-model/src/test/derived/structanyorder/ilscripts.cfg b/config-model/src/test/derived/structanyorder/ilscripts.cfg index c07f04b3021..a806bc1b712 100644 --- a/config-model/src/test/derived/structanyorder/ilscripts.cfg +++ b/config-model/src/test/derived/structanyorder/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsimplicitstruct" ilscript[].docfield[] "structfield" diff --git a/config-model/src/test/derived/tokenization/ilscripts.cfg b/config-model/src/test/derived/tokenization/ilscripts.cfg index c08b6a54c83..cad8ec81879 100644 --- a/config-model/src/test/derived/tokenization/ilscripts.cfg +++ b/config-model/src/test/derived/tokenization/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "tokenization" ilscript[].docfield[] "text" diff --git a/config-model/src/test/derived/types/ilscripts.cfg b/config-model/src/test/derived/types/ilscripts.cfg index 17bed90deb4..73befb221ce 100644 --- a/config-model/src/test/derived/types/ilscripts.cfg +++ b/config-model/src/test/derived/types/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "types" ilscript[].docfield[] "abyte" diff --git a/config-model/src/test/derived/uri_array/ilscripts.cfg b/config-model/src/test/derived/uri_array/ilscripts.cfg index 3dd97e5c11f..0dc87b513ce 100644 --- a/config-model/src/test/derived/uri_array/ilscripts.cfg +++ b/config-model/src/test/derived/uri_array/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "uri_array" ilscript[].docfield[] "my_uri" diff --git a/config-model/src/test/derived/uri_wset/ilscripts.cfg b/config-model/src/test/derived/uri_wset/ilscripts.cfg index 48e07ef9959..cc45ee5ad8f 100644 --- a/config-model/src/test/derived/uri_wset/ilscripts.cfg +++ b/config-model/src/test/derived/uri_wset/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "uri_wset" ilscript[].docfield[] "my_uri" diff --git a/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java index 8d2c98439ca..af6fb82ee7b 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.parser; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.io.reader.NamedReader; import static com.yahoo.config.model.test.TestUtil.joinLines; @@ -232,5 +233,16 @@ public class IntermediateCollectionTestCase { assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); } + @Test + void can_detect_errors_in_rank_profile_outside_schema() { + var collection = new IntermediateCollection(); + collection.addSchemaFromFile("src/test/derived/rankprofilemodularity/test.sd"); + var exception = assertThrows(ParseException.class, () -> { + collection.addRankProfileFile("test", "src/test/derived/rankprofilemodularity2/invalid_comment.profile"); + }); + var message = exception.getMessage(); + assertTrue(message.contains("Failed parsing rank-profile from 'src/test/derived/rankprofilemodularity2/invalid_comment.profile'")); + assertTrue(message.contains("Lexical error at line 3, column 6"), message); + } } diff --git a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java index 5a2dc218da7..4186e352388 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java @@ -121,6 +121,39 @@ public class SchemaParserTestCase { } @Test + void significance_can_be_parsed() throws Exception { + String input = """ + schema foo { + rank-profile significance-ranking-0 inherits default { + significance { + use-model: true + } + } + rank-profile significance-ranking-1 { + significance { + use-model: false + } + } + } + """; + + ParsedSchema schema = parseString(input); + assertEquals("foo", schema.name()); + var rplist = schema.getRankProfiles(); + assertEquals(2, rplist.size()); + + var rp0 = rplist.get(0); + assertEquals("significance-ranking-0", rp0.name()); + assertTrue(rp0.isUseSignificanceModel().isPresent()); + assertTrue(rp0.isUseSignificanceModel().get()); + + var rp1 = rplist.get(1); + assertEquals("significance-ranking-1", rp1.name()); + assertTrue(rp1.isUseSignificanceModel().isPresent()); + assertFalse(rp1.isUseSignificanceModel().get()); + } + + @Test void maxOccurrencesCanBeParsed() throws Exception { String input = joinLines ("schema foo {", @@ -137,6 +170,23 @@ public class SchemaParserTestCase { assertEquals(11, field.matchSettings().getMaxTermOccurrences().get()); } + @Test + void maxTokenLengthCanBeParsed() throws Exception { + String input = joinLines + ("schema foo {", + " document foo {", + " field bar type string {", + " indexing: summary | index", + " match { max-token-length: 11 }", + " }", + " }", + "}"); + ParsedSchema schema = parseString(input); + var field = schema.getDocument().getFields().get(0); + assertEquals("bar", field.name()); + assertEquals(11, field.matchSettings().getMaxTokenLength().get()); + } + void checkFileParses(String fileName) throws Exception { var schema = parseFile(fileName); assertNotNull(schema); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java index de99d46b9ca..355a810f5ff 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java @@ -10,6 +10,7 @@ import com.yahoo.schema.Schema; import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.AbstractSchemaTestCase; import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.MatchType; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; @@ -155,6 +156,24 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { field); } + @Test + void requireThatMaxTokenLengthIsPropagated() { + var field = new SDField("test", DataType.STRING); + field.getMatching().maxTokenLength(10); + field.parseIndexingScript("test", "{ summary | index }"); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" max-token-length:10 | summary test | index test; }", + field); + } + + @Test + void requireThatMaxTokenLengthIsPropagatedForWordMatch() { + var field = new SDField("test", DataType.STRING); + field.getMatching().maxTokenLength(10).setType(MatchType.WORD); + field.parseIndexingScript("test", "{ summary | index }"); + assertIndexingScript("{ input test | exact max-token-length:10 | summary test | index test; }", + field); + } + private static void assertIndexingScript(String expectedScript, SDField unprocessedField) { assertEquals(expectedScript, processField(unprocessedField).toString()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java index 7c4968aac84..c24fcb27dc9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java @@ -2,8 +2,24 @@ package com.yahoo.vespa.model.admin.otel; import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo; +import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.PortAllocBridge; import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.*; /** @@ -13,10 +29,45 @@ public class OpenTelemetryConfigGeneratorTest { @Test void testBuildsYaml() { - var generator = new OpenTelemetryConfigGenerator(null); - generator.addStatePorts(List.of(new StatePortInfo("localhost", 19098, "config-sentinel", "sentinel"))); + var mockZone = new Zone(SystemName.PublicCd, Environment.prod, RegionName.from("mock")); + var app = ApplicationId.from("mytenant", "myapp", "myinstance"); + var generator = new OpenTelemetryConfigGenerator(mockZone, app); + var root = new MockRoot(); + + var mockHost = new Host(root, "localhost2.local"); + var mockVersion = new com.yahoo.component.Version(8); + var mockCluster = ClusterMembership.from("container/feeding/2/3", mockVersion, Optional.empty()); + var noResource = NodeResources.unspecified(); + var mockHostSpec = new HostSpec("localhost1.local", + noResource, noResource, noResource, + mockCluster, + Optional.empty(), Optional.empty(), Optional.empty()); + var mockHostResource = new HostResource(mockHost, mockHostSpec); + var mockSvc1 = new MockService(root, "sentinel"); + mockSvc1.setHostResource(mockHostResource); + var mockPort1 = new StatePortInfo("localhost", 19098, mockSvc1); + + var mockSvc2 = new MockService(root, "searchnode"); + mockSvc2.setProp("clustername", "mycluster"); + mockSvc2.setProp("clustertype", "mockup"); + var mockPort2 = new StatePortInfo("other.host.local", 19102, mockSvc2); + + generator.addStatePorts(List.of(mockPort1, mockPort2)); String yaml = generator.generate(); + // System.err.println(">>>\n" + yaml + "\n<<<"); assertTrue(yaml.contains("sentinel")); } + static class MockService extends AbstractService { + private final String name; + public MockService(TreeConfigProducer<?> parent, String name) { + super(parent, name); + this.name = name; + } + public String getServiceName() { return name; } + public String getServiceType() { return "dummy"; } + @Override public int getPortCount() { return 0; } + @Override public void allocatePorts(int start, PortAllocBridge from) { } + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java index ae1db366c9f..2e51a425f6d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java @@ -143,6 +143,32 @@ public class ComplexFieldsValidatorTestCase { } @Test + void logs_warning_when_complex_fields_have_struct_fields_with_index_and_exact_match() throws IOException, SAXException { + var logger = new MyLogger(); + createModelAndValidate(joinLines( + "schema test {", + " document test {", + " field nesteds type array<nested> {", + " struct-field foo {", + " indexing: attribute | index", + " match {", + " exact", + " exact-terminator: '@@'", + " }", + " }", + " }", + " struct nested {", + " field foo type string {}", + " }", + " }", + "}"), logger); + assertTrue(logger.message.toString().contains("For cluster 'mycluster', schema 'test': " + + "The following complex fields have struct fields with 'indexing: index' which is " + + "not supported and has no effect: nesteds (nesteds.foo). " + + "Remove setting or change to 'indexing: attribute' if needed for matching.")); + } + + @Test void validation_passes_when_only_supported_struct_field_attributes_are_used() throws IOException, SAXException { createModelAndValidate(joinLines("search test {", " document test {", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java new file mode 100644 index 00000000000..13e91f60712 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java @@ -0,0 +1,79 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author lesters + */ +public class RestartOnDeployForLocalLLMValidatorTest { + + private static final String LOCAL_LLM_COMPONENT = RestartOnDeployForLocalLLMValidator.LOCAL_LLM_COMPONENT; + + @Test + void validate_no_restart_on_deploy() { + VespaModel current = createModel(); + VespaModel next = createModel(withComponent(LOCAL_LLM_COMPONENT)); + List<ConfigChangeAction> result = validateModel(current, next); + assertEquals(0, result.size()); + } + + @Test + void validate_restart_on_deploy() { + VespaModel current = createModel(withComponent(LOCAL_LLM_COMPONENT)); + VespaModel next = createModel(withComponent(LOCAL_LLM_COMPONENT)); + List<ConfigChangeAction> result = validateModel(current, next); + assertEquals(1, result.size()); + assertTrue(result.get(0).validationId().isEmpty()); + assertEquals("Need to restart services in cluster 'cluster1' due to use of local LLM", result.get(0).getMessage()); + } + + private static List<ConfigChangeAction> validateModel(VespaModel current, VespaModel next) { + return ValidationTester.validateChanges(new RestartOnDeployForLocalLLMValidator(), + next, + deployStateBuilder().previousModel(current).build()); + } + + private static VespaModel createModel(String component) { + var xml = """ + <services version='1.0'> + <container id='cluster1' version='1.0'> + <http> + <server id='server1' port='8080'/> + </http> + %s + </container> + </services> + """.formatted(component); + DeployState.Builder builder = deployStateBuilder(); + return new VespaModelCreatorWithMockPkg(null, xml).create(builder); + } + + private static VespaModel createModel() { + return createModel(""); + } + + private static String withComponent(String componentClass) { + return "<component id='llm' class='%s' />".formatted(componentClass); + } + + private static DeployState.Builder deployStateBuilder() { + return new DeployState.Builder().properties(new TestProperties()); + } + + private static void assertStartsWith(String expected, List<ConfigChangeAction> result) { + assertTrue(result.get(0).getMessage().startsWith(expected)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java index 00e95a34287..26e8c67a226 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java @@ -37,9 +37,6 @@ public class SignificanceModelTestCase { ApplicationContainerCluster containerCluster = vespaModel.getContainerClusters().get("container"); var significanceConfig = assertSignificancePresent(containerCluster); assertEquals(3, significanceConfig.model().size()); - assertEquals("en", significanceConfig.model().get(0).language()); - assertEquals("no", significanceConfig.model().get(1).language()); - assertEquals("ru", significanceConfig.model().get(2).language()); assertEquals("models/idf-norwegian-wiki.json.zst", modelReference(significanceConfig.model().get(1), "path").path().orElseThrow().value()); assertEquals("https://some/uri/blob.json", modelReference(significanceConfig.model().get(2), "path").url().orElseThrow().value()); diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 7333ef5a87b..a413ec7753b 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -168,7 +168,7 @@ </threadpool> <significance> - <model language="en" model-id="idf-wiki-simple-english" path="models/idf-simple-english-wiki.json.zst" /> + <model model-id="idf-wiki-simple-english" path="models/idf-simple-english-wiki.json.zst" /> </significance> </search> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index f73fec3ec68..094a7c5c003 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -25,6 +25,7 @@ public enum SystemName { /** Continuous deployment system for testing the Public system */ PublicCd(true, true), + PublicCdMigration(true, true), /** Local development system */ dev(false, false); @@ -48,6 +49,7 @@ public enum SystemName { case "main": return main; case "public": return Public; case "publiccd": return PublicCd; + case "publiccdmigration": return PublicCdMigration; default: throw new IllegalArgumentException(String.format("'%s' is not a valid system", value)); } } @@ -59,6 +61,7 @@ public enum SystemName { case main: return "main"; case Public: return "public"; case PublicCd: return "publiccd"; + case PublicCdMigration: return "publiccdmigration"; default : throw new IllegalStateException(); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java index 2db395caece..0e3e2b10531 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java @@ -20,7 +20,7 @@ public class SystemNameTest { @Test void allOf() { - assertEquals(Set.of(SystemName.cd, SystemName.PublicCd), SystemName.allOf(SystemName::isCd)); - assertEquals(Set.of(SystemName.PublicCd, SystemName.Public), SystemName.allOf(SystemName::isPublic)); + assertEquals(Set.of(SystemName.cd, SystemName.PublicCd, SystemName.PublicCdMigration), SystemName.allOf(SystemName::isCd)); + assertEquals(Set.of(SystemName.PublicCd, SystemName.Public, SystemName.PublicCdMigration), SystemName.allOf(SystemName::isPublic)); } } diff --git a/configd/src/apps/sentinel/manager.cpp b/configd/src/apps/sentinel/manager.cpp index 36bdef0dd8a..9ee259bb892 100644 --- a/configd/src/apps/sentinel/manager.cpp +++ b/configd/src/apps/sentinel/manager.cpp @@ -170,16 +170,17 @@ Manager::handleChildDeaths() } void -Manager::updateActiveFdset(fd_set *fds, int *maxNum) +Manager::updateActiveFdset(std::vector<pollfd> &fds) { - // ### _Possibly put an assert here if fd is > 1023??? - for (OutputConnection *c : _outputConnections) { + fds.clear(); + for (const OutputConnection *c : _outputConnections) { int fd = c->fd(); if (fd >= 0) { - FD_SET(fd, fds); - if (fd >= *maxNum) { - *maxNum = fd + 1; - } + fds.emplace_back(); + auto &ev = fds.back(); + ev.fd = fd; + ev.events = POLLIN; + ev.revents = 0; } } } diff --git a/configd/src/apps/sentinel/manager.h b/configd/src/apps/sentinel/manager.h index 765803b5da6..6967e078dd9 100644 --- a/configd/src/apps/sentinel/manager.h +++ b/configd/src/apps/sentinel/manager.h @@ -9,6 +9,7 @@ #include "state-api.h" #include <vespa/config-sentinel.h> #include <vespa/vespalib/net/http/state_server.h> +#include <poll.h> #include <sys/types.h> #include <sys/select.h> @@ -54,7 +55,7 @@ public: virtual ~Manager(); bool terminate(); bool doWork(); - void updateActiveFdset(fd_set *fds, int *maxNum); + void updateActiveFdset(std::vector<pollfd> &fds); }; } diff --git a/configd/src/apps/sentinel/sentinel.cpp b/configd/src/apps/sentinel/sentinel.cpp index 4f1d6019065..db9f73ea76d 100644 --- a/configd/src/apps/sentinel/sentinel.cpp +++ b/configd/src/apps/sentinel/sentinel.cpp @@ -10,7 +10,6 @@ #include <clocale> #include <string> #include <unistd.h> -#include <sys/time.h> #include <vespa/log/log.h> LOG_SETUP("sentinel.config-sentinel"); @@ -84,6 +83,7 @@ main(int argc, char **argv) } sentinel::Manager manager(environment); + std::vector<pollfd> fds; vespalib::steady_time lastTime = vespalib::steady_clock::now(); while (!stop()) { try { @@ -103,16 +103,10 @@ main(int argc, char **argv) if (vespalib::SignalHandler::CHLD.check()) { continue; } - int maxNum = 0; - fd_set fds; - FD_ZERO(&fds); - manager.updateActiveFdset(&fds, &maxNum); + manager.updateActiveFdset(fds); + constexpr int poll_timeout_ms = 100; - struct timeval tv; - tv.tv_sec = 0; - tv.tv_usec = 100000; //0.1s - - select(maxNum, &fds, nullptr, nullptr, &tv); + poll(fds.data(), fds.size(), poll_timeout_ms); vespalib::steady_time now = vespalib::steady_clock::now(); if ((now - lastTime) < 10ms) { diff --git a/configdefinitions/src/vespa/ilscripts.def b/configdefinitions/src/vespa/ilscripts.def index acb06abb755..7a286773564 100644 --- a/configdefinitions/src/vespa/ilscripts.def +++ b/configdefinitions/src/vespa/ilscripts.def @@ -3,6 +3,8 @@ namespace=vespa.configdefinition ## The maximum number of occurrences of a given term to index per field maxtermoccurrences int default=10000 +## The maximum number of characters for a token +maxtokenlength int default=1000 fieldmatchmaxlength int default=1000000 ilscript[].doctype string diff --git a/configdefinitions/src/vespa/significance.def b/configdefinitions/src/vespa/significance.def index e0cc5b4c611..8d40381a0c9 100644 --- a/configdefinitions/src/vespa/significance.def +++ b/configdefinitions/src/vespa/significance.def @@ -1,6 +1,4 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=search.significance.config -model[].language string model[].path model - diff --git a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java index b4536a1c56b..440df4f9be9 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java +++ b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; import static com.yahoo.text.Lowercase.toLowerCase; @@ -74,41 +75,52 @@ public final class CompoundName { * @param compounds the compounds of this name */ private CompoundName(String name, String [] compounds, boolean useCache) { - if (name == null) throw new NullPointerException("Name can not be null"); - - this.name = name; + this.name = Objects.requireNonNull(name, "Name can not be null"); this.lowerCasedName = toLowerCase(name); - if (compounds.length == 1 && compounds[0].isEmpty()) { - this.compounds = List.of(); - this.hashCode = 0; - rest = this; - first = this; + if (compounds.length == 1) { + if (compounds[0].isEmpty()) { + this.compounds = List.of(); + this.hashCode = 0; + rest = first = this; + return; + } + this.compounds = new ImmutableArrayList(compounds); + this.hashCode = this.compounds.hashCode(); + rest = first = empty; return; } - this.compounds = new ImmutableArrayList(compounds); - this.hashCode = this.compounds.hashCode(); - - if (compounds.length > 1) { - String restName = name.substring(compounds[0].length()+1); - if (useCache) { - rest = cache.computeIfAbsent(restName, (key) -> new CompoundName(key, Arrays.copyOfRange(compounds, 1, compounds.length), useCache)); - } else { - rest = new CompoundName(restName, Arrays.copyOfRange(compounds, 1, compounds.length), useCache); + CompoundName[] children = new CompoundName[compounds.length]; + for (int i = 0; i + 1 < children.length; i++) { + int start = 0, end = i == 0 ? -1 : children[0].name.length(); + for (int j = 0; j + i < children.length; j++) { + end += compounds[j + i].length() + 1; + if (end == start) throw new IllegalArgumentException("'" + name + "' is not a legal compound name. " + + "Consecutive, leading or trailing dots are not allowed."); + String subName = this.name.substring(start, end); + CompoundName cached = cache.get(subName); + children[j] = cached != null ? cached + : new CompoundName(subName, + this.lowerCasedName.substring(start, end), + Arrays.copyOfRange(compounds, j, j + i + 1), + i == 0 ? empty : children[j + 1], + i == 0 ? empty : children[j]); + if (useCache && cached == null) cache.put(subName, children[j]); + start += compounds[j].length() + 1; } - } else { - rest = empty; } + this.compounds = new ImmutableArrayList(compounds); + this.hashCode = this.compounds.hashCode(); + this.rest = children[1]; + this.first = children[0]; + } - if (compounds.length > 1) { - String firstName = name.substring(0, name.length() - (compounds[compounds.length-1].length()+1)); - if (useCache) { - first = cache.computeIfAbsent(firstName, (key) -> new CompoundName(key, Arrays.copyOfRange(compounds, 0, compounds.length-1), useCache)); - } else { - first = new CompoundName(firstName, Arrays.copyOfRange(compounds, 0, compounds.length-1), useCache); - } - } else { - first = empty; - } + private CompoundName(String name, String lowerCasedName, String[] compounds, CompoundName rest, CompoundName first) { + this.name = name; + this.lowerCasedName = lowerCasedName; + this.compounds = new ImmutableArrayList(compounds); + this.hashCode = this.compounds.hashCode(); + this.rest = rest; + this.first = first; } private static List<String> parse(String s) { diff --git a/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java b/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java index b5143f89c78..7523a68501f 100644 --- a/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java +++ b/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java @@ -13,7 +13,7 @@ import static org.junit.jupiter.api.Assertions.*; /** * Module local test of the basic property name building block. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class CompoundNameTestCase { @@ -30,22 +30,22 @@ public class CompoundNameTestCase { } @Test - final void testLast() { + void testLast() { assertEquals(NAME.substring(NAME.lastIndexOf('.') + 1), C_NAME.last()); } @Test - final void testFirst() { + void testFirst() { assertEquals(NAME.substring(0, NAME.indexOf('.')), C_NAME.first()); } @Test - final void testRest() { + void testRest() { verifyStrict(NAME.substring(NAME.indexOf('.') + 1), C_NAME.rest()); } @Test - final void testRestN() { + void testRestN() { verifyStrict("a.b.c.d.e", C_abcde.rest(0)); verifyStrict("b.c.d.e", C_abcde.rest(1)); verifyStrict("c.d.e", C_abcde.rest(2)); @@ -53,8 +53,9 @@ public class CompoundNameTestCase { verifyStrict("e", C_abcde.rest(4)); verifyStrict(CompoundName.empty, C_abcde.rest(5)); } + @Test - final void testFirstN() { + void testFirstN() { verifyStrict("a.b.c.d.e", C_abcde.first(5)); verifyStrict("a.b.c.d", C_abcde.first(4)); verifyStrict("a.b.c", C_abcde.first(3)); @@ -64,15 +65,32 @@ public class CompoundNameTestCase { } @Test - final void testPrefix() { - CompoundName abc = CompoundName.from("a.b.c"); - assertTrue(abc.hasPrefix(CompoundName.empty)); - assertTrue(abc.hasPrefix(CompoundName.from("a"))); - assertTrue(abc.hasPrefix(CompoundName.from("a.b"))); - assertTrue(abc.hasPrefix(CompoundName.from("a.b.c"))); + void testPrefix() { + CompoundName abcc = CompoundName.from("a.b.cc"); + assertTrue(abcc.hasPrefix(CompoundName.empty)); + assertTrue(abcc.hasPrefix(CompoundName.from("a"))); + assertTrue(abcc.hasPrefix(CompoundName.from("a.b"))); + assertTrue(abcc.hasPrefix(CompoundName.from("a.b.cc"))); - assertFalse(abc.hasPrefix(CompoundName.from("a.b.c.d"))); - assertFalse(abc.hasPrefix(CompoundName.from("a.b.d"))); + assertFalse(abcc.hasPrefix(CompoundName.from("a.b.c"))); + assertFalse(abcc.hasPrefix(CompoundName.from("a.b.c.d"))); + assertFalse(abcc.hasPrefix(CompoundName.from("a.b.d"))); + } + + @Test + void testIllegalCompound() { + assertEquals("'a.' is not a legal compound name. Names can not end with a dot.", + assertThrows(IllegalArgumentException.class, + () -> CompoundName.from("a.")) + .getMessage()); + assertEquals("'.b' is not a legal compound name. Consecutive, leading or trailing dots are not allowed.", + assertThrows(IllegalArgumentException.class, + () -> CompoundName.from(".b")) + .getMessage()); + assertEquals("'a..b' is not a legal compound name. Consecutive, leading or trailing dots are not allowed.", + assertThrows(IllegalArgumentException.class, + () -> CompoundName.from("a..b")) + .getMessage()); } @Test @@ -82,7 +100,7 @@ public class CompoundNameTestCase { } @Test - final void testSize() { + void testSize() { Splitter s = Splitter.on('.'); Iterable<String> i = s.split(NAME); int n = 0; @@ -93,23 +111,23 @@ public class CompoundNameTestCase { } @Test - final void testGet() { + void testGet() { String s = C_NAME.get(0); assertEquals(NAME.substring(0, NAME.indexOf('.')), s); } @Test - final void testIsCompound() { + void testIsCompound() { assertTrue(C_NAME.isCompound()); } @Test - final void testIsEmpty() { + void testIsEmpty() { assertFalse(C_NAME.isEmpty()); } @Test - final void testAsList() { + void testAsList() { List<String> l = C_NAME.asList(); Splitter peoplesFront = Splitter.on('.'); Iterable<String> answer = peoplesFront.split(NAME); @@ -121,7 +139,7 @@ public class CompoundNameTestCase { } @Test - final void testEqualsObject() { + void testEqualsObject() { assertNotEquals(C_NAME, NAME); assertNotEquals(C_NAME, null); verifyStrict(C_NAME, C_NAME); @@ -129,7 +147,7 @@ public class CompoundNameTestCase { } @Test - final void testEmptyNonEmpty() { + void testEmptyNonEmpty() { assertTrue(CompoundName.empty.isEmpty()); assertEquals(0, CompoundName.empty.size()); assertFalse(CompoundName.from("a").isEmpty()); @@ -140,7 +158,7 @@ public class CompoundNameTestCase { } @Test - final void testGetLowerCasedName() { + void testGetLowerCasedName() { assertEquals(Lowercase.toLowerCase(NAME), C_NAME.getLowerCasedName()); } @@ -223,4 +241,5 @@ public class CompoundNameTestCase { assertEquals("[one]", CompoundName.from("one").asList().toString()); assertEquals("[one, two, three]", CompoundName.from("one.two.three").asList().toString()); } + } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 5e66e1bb746..1c6c773afd9 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -5499,6 +5499,8 @@ "public com.yahoo.search.query.ranking.RankProperties getProperties()", "public void setListFeatures(boolean)", "public boolean getListFeatures()", + "public void setUseSignificance(boolean)", + "public boolean getUseSignificance()", "public com.yahoo.search.query.ranking.MatchPhase getMatchPhase()", "public com.yahoo.search.query.ranking.GlobalPhase getGlobalPhase()", "public com.yahoo.search.query.ranking.Matching getMatching()", @@ -8537,6 +8539,7 @@ "public com.yahoo.search.schema.RankProfile$Builder setHasSummaryFeatures(boolean)", "public com.yahoo.search.schema.RankProfile$Builder setHasRankFeatures(boolean)", "public com.yahoo.search.schema.RankProfile$Builder addInput(java.lang.String, com.yahoo.search.schema.RankProfile$InputType)", + "public com.yahoo.search.schema.RankProfile$Builder setUseSignificanceModel(boolean)", "public com.yahoo.search.schema.RankProfile build()" ], "fields" : [ ] @@ -8571,6 +8574,7 @@ "public com.yahoo.search.schema.Schema schema()", "public boolean hasSummaryFeatures()", "public boolean hasRankFeatures()", + "public boolean useSignificanceModel()", "public java.util.Map inputs()", "public boolean equals(java.lang.Object)", "public int hashCode()", diff --git a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java index f565315b775..4c39506ed96 100755 --- a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java +++ b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java @@ -33,7 +33,7 @@ import java.util.stream.Collectors; @Beta public class LLMSearcher extends Searcher { - private static Logger log = Logger.getLogger(LLMSearcher.class.getName()); + private static final Logger log = Logger.getLogger(LLMSearcher.class.getName()); private static final String API_KEY_HEADER = "X-LLM-API-KEY"; private static final String STREAM_PROPERTY = "stream"; @@ -202,7 +202,7 @@ public class LLMSearcher extends Searcher { private static class TokenStats { - private long start; + private final long start; private long timeToFirstToken; private long timeToLastToken; private long tokens = 0; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index d7fad148c8c..bfcf0af325d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -79,7 +79,7 @@ public abstract class InvokerFactory { success.add(node); } } - if ( ! cluster.isPartialGroupCoverageSufficient(success) && !acceptIncompleteCoverage) { + if ( ! cluster.isPartialGroupCoverageSufficient(group.hasSufficientCoverage(), success) && !acceptIncompleteCoverage) { return Optional.empty(); } if (invokers.isEmpty()) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index 965ce4aeb94..c7af37b3a26 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -23,7 +23,7 @@ public class Group { // Using volatile to ensure visibility for reader. // All updates are done in a single writer thread - private volatile boolean hasSufficientCoverage = true; + private volatile boolean hasSufficientCoverage = false; private volatile boolean hasFullCoverage = true; private volatile long activeDocuments = 0; private volatile long targetActiveDocuments = 0; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 56545a32831..8f83d8ef5ce 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -226,17 +226,20 @@ public class SearchCluster implements NodeManager<Node> { // With just one group sufficient coverage may not be the same as full coverage, as the // group will always be marked sufficient for use. updateSufficientCoverage(group, true); - boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.activeDocuments(), group.activeDocuments()); - trackGroupCoverageChanges(group, sufficientCoverage, group.activeDocuments()); + boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.hasSufficientCoverage(), + group.activeDocuments(), group.activeDocuments(), group.activeDocuments()); + trackGroupCoverageChanges(group, sufficientCoverage, group.activeDocuments(), group.activeDocuments()); } private void pingIterationCompletedMultipleGroups(SearchGroupsImpl groups) { groups.groups().forEach(Group::aggregateNodeValues); - long medianDocuments = groups.medianDocumentsPerGroup(); + long medianDocuments = groups.medianDocumentCount(); + long maxDocuments = groups.maxDocumentCount(); for (Group group : groups.groups()) { - boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.activeDocuments(), medianDocuments); + boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.hasSufficientCoverage(), + group.activeDocuments(), medianDocuments, maxDocuments); updateSufficientCoverage(group, sufficientCoverage); - trackGroupCoverageChanges(group, sufficientCoverage, medianDocuments); + trackGroupCoverageChanges(group, sufficientCoverage, medianDocuments, maxDocuments); } } @@ -261,7 +264,7 @@ public class SearchCluster implements NodeManager<Node> { /** * Calculate whether a subset of nodes in a group has enough coverage */ - private void trackGroupCoverageChanges(Group group, boolean fullCoverage, long medianDocuments) { + private void trackGroupCoverageChanges(Group group, boolean fullCoverage, long medianDocuments, long maxDocuments) { if ( ! hasInformationAboutAllNodes()) return; // Be silent until we know what we are talking about. boolean changed = group.fullCoverageStatusChanged(fullCoverage); if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { @@ -278,7 +281,7 @@ public class SearchCluster implements NodeManager<Node> { unresponsive.append('\n').append(node); } String message = "Cluster " + clusterId + ": " + group + " has reduced coverage: " + - "Active documents: " + group.activeDocuments() + "/" + medianDocuments + ", " + + "Active documents: " + group.activeDocuments() + "/" + maxDocuments + ", " + "Target active documents: " + group.targetActiveDocuments() + ", " + "working nodes: " + group.workingNodes() + "/" + group.nodes().size() + ", unresponsive nodes: " + (unresponsive.toString().isEmpty() ? " none" : unresponsive); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java index 85063b8ef57..0bb694f610e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java @@ -13,21 +13,30 @@ import static java.util.stream.Collectors.toCollection; import static java.util.stream.Collectors.toSet; /** - * Simple interface for groups and their nodes in the content cluster + * Simple interface for groups and their nodes in the content cluster. + * * @author baldersheim */ public interface SearchGroups { + Group get(int id); + Set<Integer> keys(); + Collection<Group> groups(); + default boolean isEmpty() { return size() == 0; } + default Set<Node> nodes() { return groups().stream().flatMap(group -> group.nodes().stream()) .sorted(comparingInt(Node::key)) .collect(toCollection(LinkedHashSet::new)); } + int size(); - boolean isPartialGroupCoverageSufficient(Collection<Node> nodes); + + boolean isPartialGroupCoverageSufficient(boolean currentCoverageSufficient, Collection<Node> nodes); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java index c49a140804c..6528c5d2ae4 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java @@ -7,14 +7,17 @@ import java.util.Collection; import java.util.Map; import java.util.Set; +/** + * @author baldersheim + */ public class SearchGroupsImpl implements SearchGroups { private final Map<Integer, Group> groups; - private final double minActivedocsPercentage; + private final double minActiveDocsPercentage; - public SearchGroupsImpl(Map<Integer, Group> groups, double minActivedocsPercentage) { + public SearchGroupsImpl(Map<Integer, Group> groups, double minActiveDocsPercentage) { this.groups = Map.copyOf(groups); - this.minActivedocsPercentage = minActivedocsPercentage; + this.minActiveDocsPercentage = minActiveDocsPercentage; } @Override public Group get(int id) { return groups.get(id); } @@ -23,23 +26,38 @@ public class SearchGroupsImpl implements SearchGroups { @Override public int size() { return groups.size(); } @Override - public boolean isPartialGroupCoverageSufficient(Collection<Node> nodes) { - if (size() == 1) - return true; - long activeDocuments = nodes.stream().mapToLong(Node::getActiveDocuments).sum(); - return isGroupCoverageSufficient(activeDocuments, medianDocumentsPerGroup()); + public boolean isPartialGroupCoverageSufficient(boolean currentIsGroupCoverageSufficient, Collection<Node> nodes) { + if (size() == 1) return true; + long groupDocumentCount = nodes.stream().mapToLong(Node::getActiveDocuments).sum(); + return isGroupCoverageSufficient(currentIsGroupCoverageSufficient, + groupDocumentCount, medianDocumentCount(), maxDocumentCount()); } - public boolean isGroupCoverageSufficient(long activeDocuments, long medianDocuments) { - if (medianDocuments <= 0) return true; - double documentCoverage = 100.0 * (double) activeDocuments / medianDocuments; - return documentCoverage >= minActivedocsPercentage; + public boolean isGroupCoverageSufficient(boolean currentIsGroupCoverageSufficient, + long groupDocumentCount, long medianDocumentCount, long maxDocumentCount) { + if (medianDocumentCount <= 0) return true; + if (currentIsGroupCoverageSufficient) { + // To take a group *out of* rotation, require that it has less active documents than the median. + // This avoids scenarios where incorrect accounting in a single group takes all other groups offline. + double documentCoverage = 100.0 * (double) groupDocumentCount / medianDocumentCount; + return documentCoverage >= minActiveDocsPercentage; + } + else { + // to put a group *in* rotation, require that it has as many documents as the largest group, + // to avoid taking groups in too early when the majority of the groups have just been added. + double documentCoverage = 100.0 * (double) groupDocumentCount / maxDocumentCount; + return documentCoverage >= minActiveDocsPercentage; + } } - public long medianDocumentsPerGroup() { + public long medianDocumentCount() { if (isEmpty()) return 0; double[] activeDocuments = groups().stream().mapToDouble(Group::activeDocuments).toArray(); return (long) Quantiles.median().computeInPlace(activeDocuments); } + public long maxDocumentCount() { + return (long)groups().stream().mapToDouble(Group::activeDocuments).max().orElse(0); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java index 01167be6b8b..fdedbdc2fd9 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java +++ b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java @@ -64,8 +64,8 @@ class Json2SingleLevelMap { } void parse(Map<String, String> map, String parent) throws IOException { - for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parent + parser.getCurrentName(); + for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parent + parser.currentName(); JsonToken token = parser.nextToken(); if ((token == JsonToken.VALUE_STRING) || (token == JsonToken.VALUE_NUMBER_FLOAT) || @@ -89,9 +89,9 @@ class Json2SingleLevelMap { } private String skipChildren(JsonParser parser, byte [] input) throws IOException { - JsonLocation start = parser.getCurrentLocation(); + JsonLocation start = parser.currentLocation(); parser.skipChildren(); - JsonLocation end = parser.getCurrentLocation(); + JsonLocation end = parser.currentLocation(); int offset = (int)start.getByteOffset() - 1; return new String(input, offset, (int)(end.getByteOffset() - offset), StandardCharsets.UTF_8); } diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index b1dd5624d18..09de1a24ef9 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -113,6 +113,8 @@ public class Ranking implements Cloneable { private SoftTimeout softTimeout = new SoftTimeout(); + private boolean useSignificance = false; + public Ranking(Query parent) { this.parent = parent; this.rankFeatures = new RankFeatures(this); @@ -217,6 +219,14 @@ public class Ranking implements Cloneable { /** Returns whether rank features should be dumped with the result of this query, default false */ public boolean getListFeatures() { return listFeatures; } + /** Set whether to use significance in ranking */ + @com.yahoo.api.annotations.Beta + public void setUseSignificance(boolean useSignificance) { this.useSignificance = useSignificance; } + + /** Returns whether to use significance in ranking */ + @com.yahoo.api.annotations.Beta + public boolean getUseSignificance() { return useSignificance; } + /** Returns the match phase rank settings of this. This is never null. */ public MatchPhase getMatchPhase() { return matchPhase; } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index c90612425fa..90d5e04d2b6 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -256,26 +256,18 @@ public class SelectParser implements Parser { } private Item buildFunctionCall(String key, Inspector value) { - switch (key) { - case WAND: - return buildWand(key, value); - case WEIGHTED_SET: - return buildWeightedSet(key, value); - case DOT_PRODUCT: - return buildDotProduct(key, value); - case GEO_LOCATION: - return buildGeoLocation(key, value); - case NEAREST_NEIGHBOR: - return buildNearestNeighbor(key, value); - case PREDICATE: - return buildPredicate(key, value); - case RANK: - return buildRank(key, value); - case WEAK_AND: - return buildWeakAnd(key, value); - default: - throw newUnexpectedArgumentException(key, DOT_PRODUCT, NEAREST_NEIGHBOR, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); - } + return switch (key) { + case WAND -> buildWand(key, value); + case WEIGHTED_SET -> buildWeightedSet(key, value); + case DOT_PRODUCT -> buildDotProduct(key, value); + case GEO_LOCATION -> buildGeoLocation(key, value); + case NEAREST_NEIGHBOR -> buildNearestNeighbor(key, value); + case PREDICATE -> buildPredicate(key, value); + case RANK -> buildRank(key, value); + case WEAK_AND -> buildWeakAnd(key, value); + default -> + throw newUnexpectedArgumentException(key, DOT_PRODUCT, NEAREST_NEIGHBOR, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); + }; } private void addItemsFromInspector(CompositeItem item, Inspector inspector){ @@ -312,15 +304,11 @@ public class SelectParser implements Parser { private HashMap<Integer, Inspector> childMap(Inspector inspector) { HashMap<Integer, Inspector> children = new HashMap<>(); if (inspector.type() == ARRAY){ - inspector.traverse((ArrayTraverser) (index, new_value) -> { - children.put(index, new_value); - }); + inspector.traverse((ArrayTraverser) children::put); } else if (inspector.type() == OBJECT){ if (inspector.field("children").valid()){ - inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> { - children.put(index, new_value); - }); + inspector.field("children").traverse((ArrayTraverser) children::put); } } return children; @@ -336,9 +324,7 @@ public class SelectParser implements Parser { private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation) { HashMap<String, Inspector> attributes = new HashMap<>(); if (annotation.type() == OBJECT){ - annotation.traverse((ObjectTraverser) (index, new_value) -> { - attributes.put(index, new_value); - }); + annotation.traverse((ObjectTraverser) attributes::put); } return attributes; } @@ -346,9 +332,7 @@ public class SelectParser implements Parser { private HashMap<String, Inspector> getAnnotationMap(Inspector inspector) { HashMap<String, Inspector> attributes = new HashMap<>(); if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ - inspector.field("attributes").traverse((ObjectTraverser) (index, new_value) -> { - attributes.put(index, new_value); - }); + inspector.field("attributes").traverse((ObjectTraverser) attributes::put); } return attributes; } @@ -487,7 +471,6 @@ public class SelectParser implements Parser { return item; } - @SuppressWarnings("deprecation") private CompositeItem buildWeakAnd(String key, Inspector value) { WeakAndItem weakAnd = new WeakAndItem(); addItemsFromInspector(weakAnd, value); @@ -576,8 +559,7 @@ public class SelectParser implements Parser { } }); } - if (out instanceof IntItem && annotations != null) { - IntItem number = (IntItem) out; + if (out instanceof IntItem number && annotations != null) { Integer hitLimit = getCappedRangeSearchParameter(annotations); if (hitLimit != null) { number.setHitLimit(hitLimit); @@ -631,12 +613,13 @@ public class SelectParser implements Parser { throw new IllegalArgumentException("The first array element under 'equals' should be a field name string " + "but was " + children.get(0)); String field = children.get(0).asString(); - switch (children.get(1).type()) { - case BOOL: return new BoolItem(children.get(1).asBool(), field); - case LONG: return new IntItem(children.get(1).asLong(), field); - default: throw new IllegalArgumentException("The second array element under 'equals' should be a boolean " + - "or int value but was " + children.get(1)); - } + return switch (children.get(1).type()) { + case BOOL -> new BoolItem(children.get(1).asBool(), field); + case LONG -> new IntItem(children.get(1).asLong(), field); + default -> + throw new IllegalArgumentException("The second array element under 'equals' should be a boolean " + + "or int value but was " + children.get(1)); + }; } private Item buildRange(String key, Inspector value) { @@ -661,15 +644,15 @@ public class SelectParser implements Parser { throw new IllegalArgumentException("Expected a numeric argument to range, but got the string '" + bound.asString() + "'"); } if (operator.equals("=")) { - bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + bounds[0] = (bound.type() == DOUBLE) ? (Number) bound.asDouble() : (Number) bound.asLong(); operators[0] = operator; equals[0] = true; } if (operator.equals(">=") || operator.equals(">")){ - bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + bounds[0] = (bound.type() == DOUBLE) ? (Number) bound.asDouble() : (Number) bound.asLong(); operators[0] = operator; } else if (operator.equals("<=") || operator.equals("<")){ - bounds[1] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + bounds[1] = (bound.type() == DOUBLE) ? (Number) bound.asDouble() : (Number) bound.asLong(); operators[1] = operator; } @@ -680,20 +663,13 @@ public class SelectParser implements Parser { } else if (operators[0] == null || operators[1] == null) { int index = (operators[0] == null) ? 1 : 0; - switch (operators[index]){ - case ">=": - range = buildGreaterThanOrEquals(field, bounds[index].toString()); - break; - case ">": - range = buildGreaterThan(field, bounds[index].toString()); - break; - case "<": - range = buildLessThan(field, bounds[index].toString()); - break; - case "<=": - range = buildLessThanOrEquals(field, bounds[index].toString()); - break; - } + range = switch (operators[index]) { + case ">=" -> buildGreaterThanOrEquals(field, bounds[index].toString()); + case ">" -> buildGreaterThan(field, bounds[index].toString()); + case "<" -> buildLessThan(field, bounds[index].toString()); + case "<=" -> buildLessThanOrEquals(field, bounds[index].toString()); + default -> range; + }; } else { range = instantiateRangeItem(bounds[0], bounds[1], field, operators[0].equals(">"), operators[1].equals("<")); @@ -890,7 +866,7 @@ public class SelectParser implements Parser { String possibleLeafFunctionName = (possibleLeafFunction.size() > 1) ? getInspectorKey(possibleLeafFunction.get(1)) : ""; if (FUNCTION_CALLS.contains(key)) { return instantiateCompositeLeaf(field, key, value); - } else if ( ! possibleLeafFunctionName.equals("")){ + } else if (!possibleLeafFunctionName.isEmpty()){ return instantiateCompositeLeaf(field, possibleLeafFunctionName, valueListFromInspector(value).get(1).field(possibleLeafFunctionName)); } else { return instantiateWordItem(field, key, value); @@ -898,24 +874,16 @@ public class SelectParser implements Parser { } private Item instantiateCompositeLeaf(String field, String key, Inspector value) { - switch (key) { - case SAME_ELEMENT: - return instantiateSameElementItem(field, key, value); - case PHRASE: - return instantiatePhraseItem(field, key, value); - case NEAR: - return instantiateNearItem(field, key, value); - case ONEAR: - return instantiateONearItem(field, key, value); - case EQUIV: - return instantiateEquivItem(field, key, value); - case FUZZY: - return instantiateFuzzyItem(field, key, value); - case ALTERNATIVES: - return instantiateWordAlternativesItem(field, key, value); - default: - throw newUnexpectedArgumentException(key, EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT); - } + return switch (key) { + case SAME_ELEMENT -> instantiateSameElementItem(field, key, value); + case PHRASE -> instantiatePhraseItem(field, key, value); + case NEAR -> instantiateNearItem(field, key, value); + case ONEAR -> instantiateONearItem(field, key, value); + case EQUIV -> instantiateEquivItem(field, key, value); + case FUZZY -> instantiateFuzzyItem(field, key, value); + case ALTERNATIVES -> instantiateWordAlternativesItem(field, key, value); + default -> throw newUnexpectedArgumentException(key, EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT); + }; } private Item instantiateWordItem(String field, String key, Inspector value) { @@ -944,8 +912,8 @@ public class SelectParser implements Parser { Preconditions.checkArgument((prefixMatch ? 1 : 0) + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2, "Only one of prefix, substring and suffix can be set."); - final TaggableItem wordItem; + WordItem wordItem; if (exactMatch) { wordItem = new ExactStringItem(wordData, fromQuery); } else if (prefixMatch) { @@ -958,13 +926,11 @@ public class SelectParser implements Parser { wordItem = new WordItem(wordData, fromQuery); } - if (wordItem instanceof WordItem) { - prepareWord(field, value, (WordItem) wordItem); - } + prepareWord(field, value, wordItem); if (language != Language.ENGLISH) - ((Item)wordItem).setLanguage(language); + wordItem.setLanguage(language); - return (Item) leafStyleSettings(getAnnotations(value), wordItem); + return leafStyleSettings(getAnnotations(value), wordItem); } private Language decideParsingLanguage(Inspector value, String wordData) { @@ -974,9 +940,8 @@ public class SelectParser implements Parser { if (language != Language.UNKNOWN) return language; Optional<Language> explicitLanguage = query.getExplicitLanguage(); - if (explicitLanguage.isPresent()) return explicitLanguage.get(); + return explicitLanguage.orElse(Language.ENGLISH); - return Language.ENGLISH; } private void prepareWord(String field, Inspector value, WordItem wordItem) { @@ -1094,7 +1059,7 @@ public class SelectParser implements Parser { Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null); if (distance != null) { - near.setDistance((int)distance); + near.setDistance(distance); } return near; } @@ -1120,7 +1085,8 @@ public class SelectParser implements Parser { private Item instantiateEquivItem(String field, String key, Inspector value) { HashMap<Integer, Inspector> children = childMap(value); - Preconditions.checkArgument(children.size() >= 2, "Expected 2 or more arguments, got %s.", children.size()); + Preconditions.checkArgument(children.size() >= 2, + "Expected 2 or more arguments, got %s.", children.size()); EquivItem equiv = new EquivItem(); equiv.setIndexName(field); @@ -1159,8 +1125,9 @@ public class SelectParser implements Parser { private Item instantiateWordAlternativesItem(String field, String key, Inspector value) { HashMap<Integer, Inspector> children = childMap(value); - Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size()); - Preconditions.checkArgument(children.get(0).type() == OBJECT, "Expected OBJECT, got %s.", children.get(0).type()); + Preconditions.checkArgument(!children.isEmpty(), "Expected 1 or more arguments, got none"); + Preconditions.checkArgument(children.get(0).type() == OBJECT, + "Expected OBJECT, got %s.", children.get(0).type()); List<WordAlternativesItem.Alternative> terms = new ArrayList<>(); diff --git a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java index a5b8d328a7a..9583e9885e7 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java +++ b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java @@ -36,6 +36,7 @@ public class RankProfile { private final String name; private final boolean hasSummaryFeatures; private final boolean hasRankFeatures; + private final boolean useSignificanceModel; private final Map<String, InputType> inputs; // Assigned when this is added to a schema @@ -45,6 +46,7 @@ public class RankProfile { this.name = builder.name; this.hasSummaryFeatures = builder.hasSummaryFeatures; this.hasRankFeatures = builder.hasRankFeatures; + this.useSignificanceModel = builder.useSignificanceModel; this.inputs = Collections.unmodifiableMap(builder.inputs); } @@ -66,6 +68,9 @@ public class RankProfile { /** Returns true if this rank profile has rank features. */ public boolean hasRankFeatures() { return hasRankFeatures; } + /** Returns true if this rank profile should use significance models. */ + public boolean useSignificanceModel() { return useSignificanceModel; } + /** Returns the inputs explicitly declared in this rank profile. */ public Map<String, InputType> inputs() { return inputs; } @@ -76,13 +81,14 @@ public class RankProfile { if ( ! other.name.equals(this.name)) return false; if ( other.hasSummaryFeatures != this.hasSummaryFeatures) return false; if ( other.hasRankFeatures != this.hasRankFeatures) return false; + if ( other.useSignificanceModel != this.useSignificanceModel) return false; if ( ! other.inputs.equals(this.inputs)) return false; return true; } @Override public int hashCode() { - return Objects.hash(name, hasSummaryFeatures, hasRankFeatures, inputs); + return Objects.hash(name, hasSummaryFeatures, hasRankFeatures, useSignificanceModel, inputs); } @Override @@ -95,6 +101,7 @@ public class RankProfile { private final String name; private boolean hasSummaryFeatures = true; private boolean hasRankFeatures = true; + private boolean useSignificanceModel = false; private final Map<String, InputType> inputs = new LinkedHashMap<>(); public Builder(String name) { @@ -116,6 +123,8 @@ public class RankProfile { return this; } + public Builder setUseSignificanceModel(boolean use) { this.useSignificanceModel = use; return this; } + public RankProfile build() { return new RankProfile(this); } diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java index d28c2db2b9e..77f27d3d411 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -22,9 +22,10 @@ class SchemaInfoConfigurer { Schema.Builder builder = new Schema.Builder(schemaInfoConfig.name()); for (var profileConfig : schemaInfoConfig.rankprofile()) { - RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()); - profileBuilder.setHasSummaryFeatures(profileConfig.hasSummaryFeatures()); - profileBuilder.setHasRankFeatures(profileConfig.hasRankFeatures()); + RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()) + .setHasSummaryFeatures(profileConfig.hasSummaryFeatures()) + .setHasRankFeatures(profileConfig.hasRankFeatures()) + .setUseSignificanceModel(profileConfig.significance().useModel()); for (var inputConfig : profileConfig.input()) profileBuilder.addInput(inputConfig.name(), RankProfile.InputType.fromSpec(inputConfig.type())); builder.add(profileBuilder.build()); diff --git a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java index 0a42bf8a259..f6025dc6ba7 100644 --- a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java @@ -14,9 +14,16 @@ import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; +import java.util.HashSet; import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; @@ -31,16 +38,49 @@ import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; public class SignificanceSearcher extends Searcher { public final static String SIGNIFICANCE = "Significance"; - private final SignificanceModelRegistry significanceModelRegistry; + private static final Logger log = Logger.getLogger(SignificanceSearcher.class.getName()); + + private final SignificanceModelRegistry significanceModelRegistry; + private final SchemaInfo schemaInfo; @Inject - public SignificanceSearcher(SignificanceModelRegistry significanceModelRegistry) { + public SignificanceSearcher(SignificanceModelRegistry significanceModelRegistry, SchemaInfo schemaInfo) { this.significanceModelRegistry = significanceModelRegistry; + this.schemaInfo = schemaInfo; } @Override public Result search(Query query, Execution execution) { + var rankProfileName = query.getRanking().getProfile(); + + // Determine significance setup per schema for the given rank profile + var perSchemaSetup = schemaInfo.newSession(query).schemas().stream() + .collect(Collectors.toMap(Schema::name, schema -> + // Fallback to disabled if the rank profile is not found in the schema + // This will result in a failure later (in a "backend searcher") anyway. + Optional.ofNullable(schema.rankProfiles().get(rankProfileName)) + .map(RankProfile::useSignificanceModel).orElse(false))); + var uniqueSetups = new HashSet<>(perSchemaSetup.values()); + + // Fail if the significance setup for the selected schemas are conflicting + if (uniqueSetups.size() > 1) { + var result = new Result(query); + result.hits().addError( + ErrorMessage.createIllegalQuery( + ("Inconsistent 'significance' configuration for the rank profile '%s' in the schemas %s. " + + "Use 'restrict' to limit the query to a subset of schemas " + + "(https://docs.vespa.ai/en/schemas.html#multiple-schemas). " + + "Specify same 'significance' configuration for all selected schemas " + + "(https://docs.vespa.ai/en/reference/schema-reference.html#significance).") + .formatted(rankProfileName, perSchemaSetup.keySet()))); + return result; + } + + if (perSchemaSetup.isEmpty()) return execution.search(query); + var useSignificanceModel = uniqueSetups.iterator().next(); + if (!useSignificanceModel) return execution.search(query); + Language language = query.getModel().getParsingLanguage(); Optional<SignificanceModel> model = significanceModelRegistry.getModel(language); diff --git a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def index 989fbb16973..086b47f5ae5 100644 --- a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def +++ b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def @@ -28,6 +28,7 @@ schema[].summaryclass[].fields[].dynamic bool default=false schema[].rankprofile[].name string schema[].rankprofile[].hasSummaryFeatures bool default=true schema[].rankprofile[].hasRankFeatures bool default=true +schema[].rankprofile[].significance.useModel bool default=false # The name of an input (query rank feature) accepted by this profile schema[].rankprofile[].input[].name string # The tensor type of an input (query rank feature) accepted by this profile diff --git a/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java b/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java index 3baa9715c34..2cc72a43f43 100755 --- a/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java +++ b/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java @@ -13,6 +13,7 @@ import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.result.EventStream; import com.yahoo.search.searchchain.Execution; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.net.URLEncoder; @@ -127,6 +128,7 @@ public class LLMSearcherTest { } @Test + @Disabled public void testAsyncGeneration() { var executor = Executors.newFixedThreadPool(2); var sb = new StringBuilder(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java index 2a9eaa86674..e7085b093f3 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java @@ -48,6 +48,19 @@ public class SearchClusterCoverageTest { } @Test + void three_groups_of_which_two_were_just_added() { + var tester = new SearchClusterTester(3, 3); + + tester.setDocsPerNode(100, 0); + tester.setDocsPerNode(80, 1); + tester.setDocsPerNode(80, 2); + tester.pingIterationCompleted(); + assertTrue(tester.group(0).hasSufficientCoverage()); + assertFalse(tester.group(1).hasSufficientCoverage()); + assertFalse(tester.group(2).hasSufficientCoverage()); + } + + @Test void three_groups_one_missing_docs_but_too_few() { var tester = new SearchClusterTester(3, 3); @@ -65,6 +78,10 @@ public class SearchClusterCoverageTest { var tester = new SearchClusterTester(3, 3); tester.setDocsPerNode(100, 0); + tester.setDocsPerNode(100, 1); + tester.setDocsPerNode(100, 2); + tester.pingIterationCompleted(); + tester.setDocsPerNode(100, 0); tester.setDocsPerNode(150, 1); tester.setDocsPerNode(100, 2); tester.pingIterationCompleted(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index 1b36c2b8151..8ac4f067876 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -200,8 +200,6 @@ public class SearchClusterTest { @Test void requireThatVipStatusIsDefaultDownWithLocalDispatch() { try (State test = new State("cluster.1", 1, HostName.getLocalhost(), "b")) { - assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); - assertFalse(test.vipStatus.isInRotation()); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); @@ -211,8 +209,6 @@ public class SearchClusterTest { @Test void requireThatVipStatusStaysUpWithLocalDispatchAndClusterSize1() { try (State test = new State("cluster.1", 1, HostName.getLocalhost())) { - assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); - assertFalse(test.vipStatus.isInRotation()); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); @@ -225,8 +221,6 @@ public class SearchClusterTest { @Test void requireThatVipStatusIsDefaultDownWithLocalDispatchAndClusterSize2() { try (State test = new State("cluster.1", 1, HostName.getLocalhost(), "otherhost")) { - assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); - assertFalse(test.vipStatus.isInRotation()); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java index 3e98b911fc8..2ba399cf42d 100644 --- a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java +++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java @@ -77,6 +77,7 @@ public class SchemaInfoTester { .addInput("query(myTensor1)", InputType.fromSpec("tensor(x[10])")) .build()) .add(new RankProfile.Builder("bOnly") + .setUseSignificanceModel(true) .addInput("query(myTensor1)", InputType.fromSpec("tensor(a{},b{})")) .build()) .build()); @@ -129,7 +130,8 @@ public class SchemaInfoTester { rankProfileInconsistentB.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(x[10])")); schemaB.rankprofile(rankProfileInconsistentB); var rankProfileBOnly = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileBOnly.name("bOnly"); + rankProfileBOnly.name("bOnly") + .significance(new SchemaInfoConfig.Schema.Rankprofile.Significance.Builder().useModel(true)); rankProfileBOnly.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); schemaB.rankprofile(rankProfileBOnly); diff --git a/container-search/src/test/java/com/yahoo/search/significance/model/en.json b/container-search/src/test/java/com/yahoo/search/significance/model/en.json index 50bae5e3451..04010959a58 100644 --- a/container-search/src/test/java/com/yahoo/search/significance/model/en.json +++ b/container-search/src/test/java/com/yahoo/search/significance/model/en.json @@ -2,13 +2,17 @@ "version" : "1.0", "id" : "test::1", "description" : "desc", - "corpus-size" : 10, - "language" : "en", - "word-count" : 4, - "frequencies" : { - "usa" : 2, - "hello": 3, - "world": 5, - "test": 2 + "languages" : { + "en": { + "description" : "english model", + "document-count" : 10, + "language" : "en", + "document-frequencies" : { + "usa" : 2, + "hello": 3, + "world": 5, + "test": 2 + } + } } } diff --git a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java index 890db3abb51..cb5722074ff 100644 --- a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java @@ -2,6 +2,7 @@ package com.yahoo.search.significance.test; import com.yahoo.component.chain.Chain; +import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.language.Language; import com.yahoo.language.significance.SignificanceModel; import com.yahoo.language.significance.SignificanceModelRegistry; @@ -10,12 +11,19 @@ import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.schema.DocumentSummary; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.significance.SignificanceSearcher; +import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.jupiter.api.Test; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import static com.yahoo.test.JunitCompat.assertEquals; @@ -29,12 +37,18 @@ public class SignificanceSearcherTest { SignificanceModelRegistry significanceModelRegistry; SignificanceSearcher searcher; - public SignificanceSearcherTest() { - HashMap<Language, Path> map = new HashMap<>(); - map.put(Language.ENGLISH, Path.of("src/test/java/com/yahoo/search/significance/model/en.json")); - significanceModelRegistry = new DefaultSignificanceModelRegistry(map); - searcher = new SignificanceSearcher(significanceModelRegistry); + public SignificanceSearcherTest() { + List<Path> models = new ArrayList<>(); + models.add( Path.of("src/test/java/com/yahoo/search/significance/model/en.json")); + + var schema = new Schema.Builder("music") + .add(new DocumentSummary.Builder("default").build()) + .add(new RankProfile.Builder("significance-ranking") + .setUseSignificanceModel(true) + .build()); + significanceModelRegistry = new DefaultSignificanceModelRegistry(models); + searcher = new SignificanceSearcher(significanceModelRegistry, new SchemaInfo(List.of(schema.build()), List.of())); } private Execution createExecution(SignificanceSearcher searcher) { @@ -49,6 +63,7 @@ public class SignificanceSearcherTest { void testSignificanceValueOnSimpleQuery() { Query q = new Query(); + q.getRanking().setProfile("significance-ranking"); AndItem root = new AndItem(); WordItem tmp; tmp = new WordItem("Hello", true); @@ -79,6 +94,7 @@ public class SignificanceSearcherTest { @Test void testSignificanceValueOnRecursiveQuery() { Query q = new Query(); + q.getRanking().setProfile("significance-ranking"); AndItem root = new AndItem(); WordItem child1 = new WordItem("hello", true); @@ -150,4 +166,36 @@ public class SignificanceSearcherTest { assertEquals(w0_1.getSignificance(), w1.getSignificance()); } + + @Test + public void failsOnConflictingSignificanceConfiguration() { + var musicSchema = new Schema.Builder("music") + .add(new DocumentSummary.Builder("default").build()) + .add(new RankProfile.Builder("significance-ranking") + .setUseSignificanceModel(true) + .build()) + .build(); + var albumSchema = new Schema.Builder("album") + .add(new DocumentSummary.Builder("default").build()) + .add(new RankProfile.Builder("significance-ranking") + .setUseSignificanceModel(false) + .build()) + .build(); + var searcher = new SignificanceSearcher( + significanceModelRegistry, new SchemaInfo(List.of(musicSchema, albumSchema), List.of())); + + var query = new Query(); + query.getRanking().setProfile("significance-ranking"); + + var result = createExecution(searcher).search(query); + assertEquals(1, result.hits().getErrorHit().errors().size()); + + var errorMessage = result.hits().getError(); + assertEquals("Inconsistent 'significance' configuration for the rank profile 'significance-ranking' in the schemas [music, album]. " + + "Use 'restrict' to limit the query to a subset of schemas " + + "(https://docs.vespa.ai/en/schemas.html#multiple-schemas). " + + "Specify same 'significance' configuration for all selected schemas " + + "(https://docs.vespa.ai/en/reference/schema-reference.html#significance).", + errorMessage.getDetailedMessage()); + } } diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index f863816dab2..b15663e0ce6 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -56,6 +56,11 @@ public class SelectTestCase { //------------------------------------------------------------------- "where" tests @Test + void testSimple() { + assertParse("{'contains' : ['title', 'madonna']}", "title:madonna"); + } + + @Test void test_contains() { ObjectNode json = jsonMapper.createObjectNode(); ArrayNode arrayNode = jsonMapper.createArrayNode(); @@ -65,16 +70,9 @@ public class SelectTestCase { } @Test - void test() { - assertParse("{'contains' : ['title', 'madonna']}", - "title:madonna"); - } - - - @Test void testDottedFieldNames() { assertParse("{ 'contains' : ['my.nested.title', 'madonna']}", - "my.nested.title:madonna"); + "my.nested.title:madonna"); } @Test @@ -360,12 +358,12 @@ public class SelectTestCase { @Test void testRaw() { Item root = parseWhere("{ \"contains\":[ \"baz\", \"yoni jo dima\" ] }").getRoot(); - assertTrue(root instanceof WordItem); + assertInstanceOf(WordItem.class, root); assertFalse(root instanceof ExactStringItem); assertEquals("yoni jo dima", ((WordItem) root).getWord()); root = parseWhere("{ \"contains\": { \"children\" : [\"baz\", \"yoni jo dima\"], \"attributes\" : {\"grammar\" : \"raw\"} } }").getRoot(); - assertTrue(root instanceof WordItem); + assertInstanceOf(WordItem.class, root); assertFalse(root instanceof ExactStringItem); assertEquals("yoni jo dima", ((WordItem) root).getWord()); } diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java index cd9ef708920..25b54267242 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java @@ -167,6 +167,7 @@ public class StreamingSearcherTestCase { Query[] queries = new Query[4]; // Increase coverage for (int i = 0; i<queries.length; i++) { Query query = new Query(queryString); + query.setTimeout(1000); if (i == 0) { } else if (i == 1) { query.getPresentation().setSummary("summary"); diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 10aa71c1d90..f09ba696938 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -33,8 +33,8 @@ <!-- DO NOT UPGRADE THESE TO A NEW MAJOR VERSION WITHOUT CHECKING FOR BINARY COMPATIBILITY --> <aopalliance.vespa.version>1.0</aopalliance.vespa.version> - <error-prone-annotations.vespa.version>2.26.1</error-prone-annotations.vespa.version> - <guava.vespa.version>33.1.0-jre</guava.vespa.version> + <error-prone-annotations.vespa.version>2.27.1</error-prone-annotations.vespa.version> + <guava.vespa.version>33.2.0-jre</guava.vespa.version> <guice.vespa.version>6.0.0</guice.vespa.version> <j2objc-annotations.vespa.version>3.0.0</j2objc-annotations.vespa.version> <jackson2.vespa.version>2.16.2</jackson2.vespa.version> @@ -68,8 +68,8 @@ <assertj.vespa.version>3.25.3</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <aws-sdk.vespa.version>1.12.708</aws-sdk.vespa.version> - <athenz.vespa.version>1.11.57</athenz.vespa.version> + <aws-sdk.vespa.version>1.12.718</aws-sdk.vespa.version> + <athenz.vespa.version>1.11.58</athenz.vespa.version> <!-- Athenz END --> <!-- WARNING: If you change curator version, you also need to update @@ -79,12 +79,12 @@ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g' --> <bouncycastle.vespa.version>1.78.1</bouncycastle.vespa.version> - <byte-buddy.vespa.version>1.14.14</byte-buddy.vespa.version> + <byte-buddy.vespa.version>1.14.15</byte-buddy.vespa.version> <checker-qual.vespa.version>3.38.0</checker-qual.vespa.version> <commons-beanutils.vespa.version>1.9.4</commons-beanutils.vespa.version> - <commons-codec.vespa.version>1.16.1</commons-codec.vespa.version> + <commons-codec.vespa.version>1.17.0</commons-codec.vespa.version> <commons-collections.vespa.version>3.2.2</commons-collections.vespa.version> - <commons-csv.vespa.version>1.10.0</commons-csv.vespa.version> + <commons-csv.vespa.version>1.11.0</commons-csv.vespa.version> <commons-digester.vespa.version>3.2</commons-digester.vespa.version> <commons-io.vespa.version>2.16.1</commons-io.vespa.version> <commons-lang3.vespa.version>3.14.0</commons-lang3.vespa.version> @@ -102,7 +102,7 @@ <felix.log.vespa.version>1.3.0</felix.log.vespa.version> <findbugs.vespa.version>3.0.2</findbugs.vespa.version> <!-- Should be kept in sync with guava --> <hamcrest.vespa.version>2.2</hamcrest.vespa.version> - <hdrhistogram.vespa.version>2.1.12</hdrhistogram.vespa.version> + <hdrhistogram.vespa.version>2.2.1</hdrhistogram.vespa.version> <huggingface.vespa.version>0.27.0</huggingface.vespa.version> <icu4j.vespa.version>75.1</icu4j.vespa.version> <java-jjwt.vespa.version>0.11.5</java-jjwt.vespa.version> @@ -117,7 +117,7 @@ <junit.vespa.version>5.10.2</junit.vespa.version> <junit.platform.vespa.version>1.10.2</junit.platform.vespa.version> <junit4.vespa.version>4.13.2</junit4.vespa.version> - <kherud.llama.vespa.version>3.0.0</kherud.llama.vespa.version> + <kherud.llama.vespa.version>3.0.2</kherud.llama.vespa.version> <luben.zstd.vespa.version>1.5.6-3</luben.zstd.vespa.version> <lucene.vespa.version>9.10.0</lucene.vespa.version> <maven-archiver.vespa.version>3.6.2</maven-archiver.vespa.version> @@ -128,7 +128,7 @@ <mojo-executor.vespa.version>2.4.0</mojo-executor.vespa.version> <netty.vespa.version>4.1.109.Final</netty.vespa.version> <netty-tcnative.vespa.version>2.0.65.Final</netty-tcnative.vespa.version> - <onnxruntime.vespa.version>1.17.1</onnxruntime.vespa.version> + <onnxruntime.vespa.version>1.17.3</onnxruntime.vespa.version> <opennlp.vespa.version>2.3.3</opennlp.vespa.version> <opentest4j.vespa.version>1.3.0</opentest4j.vespa.version> <org.json.vespa.version>20240303</org.json.vespa.version> @@ -146,7 +146,7 @@ <surefire.vespa.version>3.2.5</surefire.vespa.version> <velocity.vespa.version>2.3</velocity.vespa.version> <velocity.tools.vespa.version>3.1</velocity.tools.vespa.version> - <wiremock.vespa.version>3.5.2</wiremock.vespa.version> + <wiremock.vespa.version>3.5.4</wiremock.vespa.version> <woodstox.vespa.version>6.6.2</woodstox.vespa.version> <stax2-api.vespa.version>4.2.2</stax2-api.vespa.version> <xerces.vespa.version>2.12.2</xerces.vespa.version> @@ -171,17 +171,17 @@ <maven-compiler-plugin.vespa.version>3.13.0</maven-compiler-plugin.vespa.version> <maven-core.vespa.version>3.9.6</maven-core.vespa.version> <maven-dependency-plugin.vespa.version>3.6.1</maven-dependency-plugin.vespa.version> - <maven-deploy-plugin.vespa.version>3.1.1</maven-deploy-plugin.vespa.version> + <maven-deploy-plugin.vespa.version>3.1.2</maven-deploy-plugin.vespa.version> <maven-enforcer-plugin.vespa.version>3.4.1</maven-enforcer-plugin.vespa.version> <maven-failsafe-plugin.vespa.version>3.2.5</maven-failsafe-plugin.vespa.version> <maven-gpg-plugin.vespa.version>3.2.4</maven-gpg-plugin.vespa.version> - <maven-install-plugin.vespa.version>3.1.1</maven-install-plugin.vespa.version> + <maven-install-plugin.vespa.version>3.1.2</maven-install-plugin.vespa.version> <maven-jar-plugin.vespa.version>3.4.1</maven-jar-plugin.vespa.version> <maven-javadoc-plugin.vespa.version>3.6.3</maven-javadoc-plugin.vespa.version> <maven-plugin-api.vespa.version>${maven-core.vespa.version}</maven-plugin-api.vespa.version> - <maven-plugin-tools.vespa.version>3.12.0</maven-plugin-tools.vespa.version> + <maven-plugin-tools.vespa.version>3.13.0</maven-plugin-tools.vespa.version> <maven-resources-plugin.vespa.version>3.3.1</maven-resources-plugin.vespa.version> - <maven-resolver.vespa.version>1.9.19</maven-resolver.vespa.version> + <maven-resolver.vespa.version>1.9.20</maven-resolver.vespa.version> <maven-shade-plugin.vespa.version>3.5.3</maven-shade-plugin.vespa.version> <maven-site-plugin.vespa.version>3.12.1</maven-site-plugin.vespa.version> <maven-source-plugin.vespa.version>3.3.1</maven-source-plugin.vespa.version> diff --git a/dist/vespa.spec b/dist/vespa.spec index 33c969ce1a6..2e6e3c042d2 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -33,11 +33,13 @@ %define _defattr_is_vespa_vespa 0 %define _command_cmake cmake3 %global _vespa_abseil_cpp_version 20240116.1 -%global _vespa_build_depencencies_version 1.3.0 +%global _vespa_build_depencencies_version 1.3.2 %global _vespa_gtest_version 1.14.0 %global _vespa_protobuf_version 5.26.1 +%global _vespa_openblas_version 0.3.27 %global _use_vespa_abseil_cpp 1 %global _use_vespa_protobuf 1 +%global _use_vespa_openblas 1 Name: vespa Version: _VESPA_VERSION_ @@ -77,7 +79,6 @@ Requires: zstd %define _devtoolset_enable /opt/rh/gcc-toolset/enable %define _use_vespa_gtest 1 -%define _use_vespa_openblas 1 %define _use_vespa_openssl 1 %if 0%{?centos} || 0%{?rocky} || 0%{?oraclelinux} @@ -157,17 +158,13 @@ Requires: vespa-xxhash >= 0.8.1 Requires: xxhash-libs >= 0.8.1 %endif %if 0%{?el8} -Requires: vespa-openssl >= 3.1.4 +Requires: vespa-openssl >= 3.1.5 %else Requires: openssl-libs %endif Requires: vespa-lz4 >= 1.9.4-1 -Requires: vespa-libzstd >= 1.5.4-1 -%if 0%{?el8} -Requires: vespa-openblas >= 0.3.26 -%else -Requires: openblas-serial -%endif +Requires: vespa-libzstd >= 1.5.6-1 +Requires: vespa-openblas >= %{_vespa_openblas_version} %if 0%{?amzn2023} Requires: vespa-re2 = 20210801 %else @@ -188,7 +185,7 @@ Summary: Vespa - The open big data serving engine - C++ libraries Requires: %{name}-base-libs = %{version}-%{release} Requires: libicu %if 0%{?el8} -Requires: vespa-openssl >= 3.1.4 +Requires: vespa-openssl >= 3.1.5 %else Requires: openssl-libs %endif @@ -204,10 +201,8 @@ Requires: vespa-protobuf = %{_vespa_protobuf_version} Requires: vespa-protobuf = %{_vespa_protobuf_version} Requires: llvm-libs %endif -Requires: vespa-onnxruntime = 1.17.1 -%if 0%{?el8} || 0%{?el9} -Requires: vespa-jllama = 3.0.0 -%endif +Requires: vespa-onnxruntime = 1.17.3 +Requires: vespa-jllama >= 3.0.1 %description libs diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java index 86b0a2e78ad..3088083912b 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java @@ -72,7 +72,7 @@ public class ScriptManager { Map<String, Map<String, DocumentScript>> documentFieldScripts = new HashMap<>(config.ilscript().size()); ScriptParserContext parserContext = new ScriptParserContext(linguistics, embedders); parserContext.getAnnotatorConfig().setMaxTermOccurrences(config.maxtermoccurrences()); - parserContext.getAnnotatorConfig().setMaxTokenLength(config.fieldmatchmaxlength()); + parserContext.getAnnotatorConfig().setMaxTokenizeLength(config.fieldmatchmaxlength()); for (IlscriptsConfig.Ilscript ilscript : config.ilscript()) { DocumentType documentType = docTypeMgr.getDocumentType(ilscript.doctype()); diff --git a/document/src/main/java/com/yahoo/document/DocumentUpdate.java b/document/src/main/java/com/yahoo/document/DocumentUpdate.java index 20d9b352d2d..d8d8c02c43b 100644 --- a/document/src/main/java/com/yahoo/document/DocumentUpdate.java +++ b/document/src/main/java/com/yahoo/document/DocumentUpdate.java @@ -188,13 +188,8 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP return Collections.unmodifiableCollection(fieldPathUpdates); } - /** Returns the type of the document this updates - * - * @return The document type of the document - */ - public DocumentType getDocumentType() { - return documentType; - } + /** Returns the type of the document this updates. */ + public DocumentType getDocumentType() { return documentType; } /** * Sets the document type. Use only while deserializing - changing the document type after creation diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index 358c0cb65e4..9c621c033bd 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -105,7 +105,7 @@ public class JsonReader { String condition = null; ParsedDocumentOperation operation = null; while (JsonToken.END_OBJECT != parser.nextValue()) { - switch (parser.getCurrentName()) { + switch (parser.currentName()) { case FIELDS -> { documentParseInfo.fieldsBuffer = new LazyTokenBuffer(parser); VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields()); @@ -177,7 +177,7 @@ public class JsonReader { state = END_OF_FEED; throw new IllegalArgumentException(r); } - if ( ! documentParseInfo.isPresent()) { + if (documentParseInfo.isEmpty()) { state = END_OF_FEED; return null; } diff --git a/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java b/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java index 0fbdd0b28c7..53ddacf6cc3 100644 --- a/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java @@ -33,7 +33,7 @@ public class LazyTokenBuffer extends TokenBuffer { public Supplier<Token> lookahead() { return new Supplier<>() { int localNesting = nesting(); - Supplier<Token> buffered = LazyTokenBuffer.super.lookahead(); + final Supplier<Token> buffered = LazyTokenBuffer.super.lookahead(); @Override public Token get() { if (localNesting == 0) return null; @@ -54,7 +54,7 @@ public class LazyTokenBuffer extends TokenBuffer { JsonToken token = parser.nextValue(); if (token == null) throw new IllegalStateException("no more JSON tokens"); - return new Token(token, parser.getCurrentName(), parser.getText()); + return new Token(token, parser.currentName(), parser.getText()); } catch (IOException e) { throw new IllegalArgumentException("failed reading document JSON", e); diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index 3a48f71c4cd..c5c022370bf 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -99,7 +99,7 @@ public class TokenBuffer { } int addFromParser(JsonParser tokens) throws IOException { - add(tokens.currentToken(), tokens.getCurrentName(), tokens.getText()); + add(tokens.currentToken(), tokens.currentName(), tokens.getText()); return nestingOffset(tokens.currentToken()); } diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java index 77e11dcf2a8..c5bcd356c94 100644 --- a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java +++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java @@ -61,7 +61,7 @@ public class DocumentParser { private boolean parseOneItem(DocumentParseInfo documentParseInfo, boolean docIdAndOperationIsSetExternally) throws IOException { parser.nextValue(); processIndent(); - if (parser.getCurrentName() == null) return false; + if (parser.currentName() == null) return false; if (indentLevel == 1L) { handleIdentLevelOne(documentParseInfo, docIdAndOperationIsSetExternally); } else if (indentLevel == 2L) { @@ -85,17 +85,18 @@ public class DocumentParser { private void handleIdentLevelOne(DocumentParseInfo documentParseInfo, boolean docIdAndOperationIsSetExternally) throws IOException { - JsonToken currentToken = parser.getCurrentToken(); + JsonToken currentToken = parser.currentToken(); + String currentName = parser.currentName(); if ((currentToken == JsonToken.VALUE_TRUE || currentToken == JsonToken.VALUE_FALSE) && - CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { + CREATE_IF_NON_EXISTENT.equals(currentName)) { documentParseInfo.create = Optional.of(currentToken == JsonToken.VALUE_TRUE); - } else if (currentToken == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) { + } else if (currentToken == JsonToken.VALUE_STRING && CONDITION.equals(currentName)) { documentParseInfo.condition = Optional.of(parser.getText()); } else if (currentToken == JsonToken.VALUE_STRING) { // Value is expected to be set in the header not in the document. Ignore any unknown field // as well. if (! docIdAndOperationIsSetExternally) { - documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName()); + documentParseInfo.operationType = operationNameToOperationType(currentName); documentParseInfo.documentId = new DocumentId(parser.getText()); } } @@ -104,7 +105,7 @@ public class DocumentParser { private void handleIdentLevelTwo(DocumentParseInfo documentParseInfo) { try { // "fields" opens a dictionary and is therefore on level two which might be surprising. - if (parser.currentToken() == JsonToken.START_OBJECT && FIELDS.equals(parser.getCurrentName())) { + if (parser.currentToken() == JsonToken.START_OBJECT && FIELDS.equals(parser.currentName())) { documentParseInfo.fieldsBuffer.bufferObject(parser); processIndent(); } diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index 7be50b58b61..bd977520b66 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -270,7 +270,6 @@ public class DocumentUpdateTestCase { DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("id:ns:my_type::foo:")); update.addFieldUpdate(FieldUpdate.createAssign(field, new IntegerFieldValue(1))); update.addFieldUpdate(FieldUpdate.createAssign(field, new IntegerFieldValue(2))); - assertEquals(1, update.fieldUpdates().size()); FieldUpdate fieldUpdate = update.getFieldUpdate(field); assertNotNull(fieldUpdate); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories80.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories80.java index 778eaeda5f0..fb78e291c7c 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories80.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories80.java @@ -387,6 +387,8 @@ abstract class RoutableFactories80 { if (apiMsg.getCondition().isPresent()) { builder.setCondition(toProtoTasCondition(apiMsg.getCondition())); } + builder.setCreateIfMissing(apiMsg.createIfMissing() ? DocapiFeed.UpdateDocumentRequest.CreateIfMissing.TRUE + : DocapiFeed.UpdateDocumentRequest.CreateIfMissing.FALSE); return builder.build(); }) .decoderWithRepo(DocapiFeed.UpdateDocumentRequest.parser(), (protoMsg, repo) -> { @@ -396,6 +398,8 @@ abstract class RoutableFactories80 { if (protoMsg.hasCondition()) { msg.setCondition(fromProtoTasCondition(protoMsg.getCondition())); } + // We ignore the createIfMissing field here since it can always be fetched eagerly + // from the DocumentUpdate instance itself. return msg; }) .build(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentMessage.java index 3fb14664628..b8609cd42b8 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentMessage.java @@ -166,4 +166,10 @@ public class UpdateDocumentMessage extends TestAndSetMessage { public void setCondition(TestAndSetCondition condition) { this.update.setCondition(condition); } + + boolean createIfMissing() { + deserialize(); + return update.getCreateIfNonExistent(); + } + } diff --git a/documentapi/src/protobuf/docapi_feed.proto b/documentapi/src/protobuf/docapi_feed.proto index 8d15fd9a536..702695ef6d8 100644 --- a/documentapi/src/protobuf/docapi_feed.proto +++ b/documentapi/src/protobuf/docapi_feed.proto @@ -39,11 +39,18 @@ message PutDocumentResponse { } message UpdateDocumentRequest { + enum CreateIfMissing { + UNSPECIFIED = 0; // Legacy fallback: must deserialize `update` to find flag value + TRUE = 1; + FALSE = 2; + } + // Note: update contains embedded document ID DocumentUpdate update = 1; TestAndSetCondition condition = 2; uint64 expected_old_timestamp = 3; uint64 force_assign_timestamp = 4; + CreateIfMissing create_if_missing = 5; } message UpdateDocumentResponse { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/Messages60TestCase.java index 42f200a0b6b..803b225bb0d 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/Messages60TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/Messages60TestCase.java @@ -504,6 +504,7 @@ public class Messages60TestCase extends MessagesTestBase { assertEquals(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp()); assertEquals(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp()); assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); + assertFalse(msg.createIfMissing()); } } } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/Messages80TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/Messages80TestCase.java index 943d9fddb26..d6a837229a9 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/Messages80TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/Messages80TestCase.java @@ -176,8 +176,52 @@ public class Messages80TestCase extends MessagesTestBase { } class UpdateDocumentMessageTest implements RunnableTest { - @Override - public void run() { + + UpdateDocumentMessage makeUpdateWithCreateIfMissing(boolean createIfMissing) { + var docType = protocol.getDocumentTypeManager().getDocumentType("testdoc"); + var update = new DocumentUpdate(docType, new DocumentId("id:ns:testdoc::")); + update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0")); + update.setCreateIfNonExistent(createIfMissing); + + var msg = new UpdateDocumentMessage(update); + msg.setNewTimestamp(777); + msg.setOldTimestamp(666); + msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); + return msg; + } + + void testLegacyCreateIfMissingFlagCanBeDeserializedFromDocumentUpdate() { + // Legacy binary files were created _prior_ to the createIfMissing flag being + // written as part of the serialization process. + forEachLanguage((lang) -> { + var msg = (UpdateDocumentMessage) deserialize( + "UpdateDocumentMessage-legacy-no-create-if-missing", + DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang); + assertFalse(msg.createIfMissing()); + + msg = (UpdateDocumentMessage) deserialize( + "UpdateDocumentMessage-legacy-with-create-if-missing", + DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang); + assertTrue(msg.createIfMissing()); + }); + } + + void checkDeserialization(Language lang, String name, boolean expectedCreate) { + var msg = (UpdateDocumentMessage) deserialize(name, DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang); + assertEquals(expectedCreate, msg.createIfMissing()); + }; + + void testCreateIfMissingFlagIsPropagated() { + serialize("UpdateDocumentMessage-no-create-if-missing", makeUpdateWithCreateIfMissing(false)); + serialize("UpdateDocumentMessage-with-create-if-missing", makeUpdateWithCreateIfMissing(true)); + + forEachLanguage((lang) -> { + checkDeserialization(lang, "UpdateDocumentMessage-no-create-if-missing", false); + checkDeserialization(lang, "UpdateDocumentMessage-with-create-if-missing", true); + }); + } + + void testAllUpdateFieldsArePropagated() { var docType = protocol.getDocumentTypeManager().getDocumentType("testdoc"); var update = new DocumentUpdate(docType, new DocumentId("id:ns:testdoc::")); update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0")); @@ -197,6 +241,13 @@ public class Messages80TestCase extends MessagesTestBase { assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); }); } + + @Override + public void run() { + testAllUpdateFieldsArePropagated(); + testLegacyCreateIfMissingFlagCanBeDeserializedFromDocumentUpdate(); + testCreateIfMissingFlagIsPropagated(); + } } class UpdateDocumentReplyTest implements RunnableTest { diff --git a/documentapi/src/tests/messages/messages60test.cpp b/documentapi/src/tests/messages/messages60test.cpp index 2af523ce8e9..861189a2d33 100644 --- a/documentapi/src/tests/messages/messages60test.cpp +++ b/documentapi/src/tests/messages/messages60test.cpp @@ -406,7 +406,7 @@ TEST_F(Messages60Test, testUpdateDocumentMessage) { msg.setNewTimestamp(777u); msg.setCondition(TestAndSetCondition("There's just one condition")); - EXPECT_EQ(sizeof(TestAndSetMessage) + 32, sizeof(UpdateDocumentMessage)); + EXPECT_EQ(sizeof(TestAndSetMessage) + 40, sizeof(UpdateDocumentMessage)); EXPECT_EQ(MESSAGE_BASE_LENGTH + 93u + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { diff --git a/documentapi/src/tests/messages/messages80test.cpp b/documentapi/src/tests/messages/messages80test.cpp index c61b1575dac..093e44ed6b1 100644 --- a/documentapi/src/tests/messages/messages80test.cpp +++ b/documentapi/src/tests/messages/messages80test.cpp @@ -29,7 +29,7 @@ TEST(MessagesTest, concrete_types_have_expected_sizes) { EXPECT_EQ(sizeof(PutDocumentMessage), sizeof(TestAndSetMessage) + 32); EXPECT_EQ(sizeof(WriteDocumentReply), 112u); EXPECT_EQ(sizeof(UpdateDocumentReply), 120u); - EXPECT_EQ(sizeof(UpdateDocumentMessage), sizeof(TestAndSetMessage) + 32); + EXPECT_EQ(sizeof(UpdateDocumentMessage), sizeof(TestAndSetMessage) + 40); EXPECT_EQ(sizeof(RemoveDocumentMessage), sizeof(TestAndSetMessage) + 104); EXPECT_EQ(sizeof(RemoveDocumentReply), 120u); } @@ -42,6 +42,14 @@ struct Messages80Test : MessageFixture { } void try_visitor_reply(const std::string& filename, uint32_t type); + + void check_update_create_flag(uint32_t lang, const std::string& name, bool expected_create, bool expected_cached) { + auto obj = deserialize(name, DocumentProtocol::MESSAGE_UPDATEDOCUMENT, lang); + ASSERT_TRUE(obj); + auto& msg = dynamic_cast<UpdateDocumentMessage&>(*obj); + EXPECT_EQ(msg.has_cached_create_if_missing(), expected_cached); + EXPECT_EQ(msg.create_if_missing(), expected_create); + }; }; namespace { @@ -180,6 +188,48 @@ TEST_F(Messages80Test, update_document_message) { } } +TEST_F(Messages80Test, update_create_if_missing_flag_can_be_read_from_legacy_update_propagation) { + // Legacy binary files were created _prior_ to the create_if_missing flag being + // written as part of the serialization process. + for (auto lang : languages()) { + check_update_create_flag(lang, "UpdateDocumentMessage-legacy-no-create-if-missing", false, false); + check_update_create_flag(lang, "UpdateDocumentMessage-legacy-with-create-if-missing", true, false); + } +} + +TEST_F(Messages80Test, update_create_if_missing_flag_is_propagated) { + const DocumentTypeRepo& repo = type_repo(); + const document::DocumentType& docType = *repo.getDocumentType("testdoc"); + + auto make_update_msg = [&](bool create_if_missing, bool cache_flag) { + auto doc_update = std::make_shared<document::DocumentUpdate>(repo, docType, document::DocumentId("id:ns:testdoc::")); + doc_update->addFieldPathUpdate(std::make_unique<document::RemoveFieldPathUpdate>("intfield", "testdoc.intfield > 0")); + doc_update->setCreateIfNonExistent(create_if_missing); + auto msg = std::make_shared<UpdateDocumentMessage>(std::move(doc_update)); + msg->setOldTimestamp(666u); + msg->setNewTimestamp(777u); + msg->setCondition(TestAndSetCondition("There's just one condition")); + if (cache_flag) { + msg->set_cached_create_if_missing(create_if_missing); + } + return msg; + }; + + serialize("UpdateDocumentMessage-no-create-if-missing", *make_update_msg(false, true)); + serialize("UpdateDocumentMessage-with-create-if-missing", *make_update_msg(true, true)); + + for (auto lang : languages()) { + check_update_create_flag(lang, "UpdateDocumentMessage-no-create-if-missing", false, true); + check_update_create_flag(lang, "UpdateDocumentMessage-with-create-if-missing", true, true); + } + // The Java protocol implementation always serializes with a cached create-flag, + // but the C++ side does it conditionally. So these files are only checked for C++. + serialize("UpdateDocumentMessage-no-create-if-missing-uncached", *make_update_msg(false, false)); + serialize("UpdateDocumentMessage-with-create-if-missing-uncached", *make_update_msg(true, false)); + check_update_create_flag(LANG_CPP, "UpdateDocumentMessage-no-create-if-missing-uncached", false, false); + check_update_create_flag(LANG_CPP, "UpdateDocumentMessage-with-create-if-missing-uncached", true, false); +} + TEST_F(Messages80Test, update_document_reply) { UpdateDocumentReply reply; reply.setWasFound(true); diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp index 3c0ffa33060..d416e587ed6 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp @@ -5,6 +5,7 @@ #include <vespa/documentapi/messagebus/documentprotocol.h> #include <vespa/document/update/documentupdate.h> #include <vespa/vespalib/util/exceptions.h> +#include <cassert> namespace documentapi { @@ -12,14 +13,16 @@ UpdateDocumentMessage::UpdateDocumentMessage() : TestAndSetMessage(), _documentUpdate(), _oldTime(0), - _newTime(0) + _newTime(0), + _create_if_missing() {} UpdateDocumentMessage::UpdateDocumentMessage(document::DocumentUpdate::SP documentUpdate) : TestAndSetMessage(), _documentUpdate(), _oldTime(0), - _newTime(0) + _newTime(0), + _create_if_missing() { setDocumentUpdate(std::move(documentUpdate)); } @@ -59,4 +62,14 @@ UpdateDocumentMessage::setDocumentUpdate(document::DocumentUpdate::SP documentUp _documentUpdate = std::move(documentUpdate); } +bool +UpdateDocumentMessage::create_if_missing() const +{ + if (_create_if_missing.has_value()) { + return *_create_if_missing; + } + assert(_documentUpdate); + return _documentUpdate->getCreateIfNonExistent(); +} + } diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h index 55aa0bf8ae4..e4a528dacdd 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h @@ -2,6 +2,7 @@ #pragma once #include "testandsetmessage.h" +#include <optional> namespace document { class DocumentUpdate; } @@ -10,9 +11,10 @@ namespace documentapi { class UpdateDocumentMessage : public TestAndSetMessage { private: using DocumentUpdateSP = std::shared_ptr<document::DocumentUpdate>; - DocumentUpdateSP _documentUpdate; - uint64_t _oldTime; - uint64_t _newTime; + DocumentUpdateSP _documentUpdate; + uint64_t _oldTime; + uint64_t _newTime; + std::optional<bool> _create_if_missing; protected: DocumentReply::UP doCreateReply() const override; @@ -28,21 +30,21 @@ public: * Constructs a new document message for deserialization. */ UpdateDocumentMessage(); - ~UpdateDocumentMessage(); + ~UpdateDocumentMessage() override; /** * Constructs a new document update message. * * @param documentUpdate The document update to perform. */ - UpdateDocumentMessage(DocumentUpdateSP documentUpdate); + explicit UpdateDocumentMessage(DocumentUpdateSP documentUpdate); /** * Returns the document update to perform. * * @return The update. */ - DocumentUpdateSP stealDocumentUpdate() const { return std::move(_documentUpdate); } + [[nodiscard]] DocumentUpdateSP stealDocumentUpdate() const { return std::move(_documentUpdate); } const document::DocumentUpdate & getDocumentUpdate() const { return *_documentUpdate; } /** * Sets the document update to perform. @@ -56,21 +58,21 @@ public: * * @return The document timestamp. */ - uint64_t getOldTimestamp() const { return _oldTime; } + [[nodiscard]] uint64_t getOldTimestamp() const noexcept { return _oldTime; } /** * Sets the timestamp required for this update to be applied. * * @param time The timestamp to set. */ - void setOldTimestamp(uint64_t time) { _oldTime = time; } + void setOldTimestamp(uint64_t time) noexcept { _oldTime = time; } /** * Returns the timestamp to assign to the updated document. * * @return The document timestamp. */ - uint64_t getNewTimestamp() const { return _newTime; } + [[nodiscard]] uint64_t getNewTimestamp() const noexcept { return _newTime; } /** * Sets the timestamp to assign to the updated document. @@ -79,6 +81,18 @@ public: */ void setNewTimestamp(uint64_t time) { _newTime = time; } + void set_cached_create_if_missing(bool create) noexcept { + _create_if_missing = create; + } + + [[nodiscard]] bool has_cached_create_if_missing() const noexcept { + return _create_if_missing.has_value(); + } + // Note: iff has_cached_create_if_missing() == false, this will trigger a deserialization of the + // underlying DocumentUpdate instance, which might throw an exception on deserialization failure. + // Otherwise, this is noexcept. + [[nodiscard]] bool create_if_missing() const; + bool hasSequenceId() const override; uint64_t getSequenceId() const override; uint32_t getType() const override; diff --git a/documentapi/src/vespa/documentapi/messagebus/routable_factories_8.cpp b/documentapi/src/vespa/documentapi/messagebus/routable_factories_8.cpp index f9782e1abd9..9ef54932f68 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routable_factories_8.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routable_factories_8.cpp @@ -312,6 +312,10 @@ std::shared_ptr<IRoutableFactory> RoutableFactories80::update_document_message_f } dest.set_expected_old_timestamp(src.getOldTimestamp()); dest.set_force_assign_timestamp(src.getNewTimestamp()); + if (src.has_cached_create_if_missing()) { + dest.set_create_if_missing(src.create_if_missing() ? protobuf::UpdateDocumentRequest_CreateIfMissing_TRUE + : protobuf::UpdateDocumentRequest_CreateIfMissing_FALSE); + } }, [type_repo = std::move(repo)](const protobuf::UpdateDocumentRequest& src) { auto msg = std::make_unique<UpdateDocumentMessage>(); @@ -321,6 +325,9 @@ std::shared_ptr<IRoutableFactory> RoutableFactories80::update_document_message_f } msg->setOldTimestamp(src.expected_old_timestamp()); msg->setNewTimestamp(src.force_assign_timestamp()); + if (src.create_if_missing() != protobuf::UpdateDocumentRequest_CreateIfMissing_UNSPECIFIED) { + msg->set_cached_create_if_missing(src.create_if_missing() == protobuf::UpdateDocumentRequest_CreateIfMissing_TRUE); + } return msg; } ); diff --git a/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-legacy-no-create-if-missing.dat b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-legacy-no-create-if-missing.dat Binary files differnew file mode 100644 index 00000000000..f1ceef0e51a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-legacy-no-create-if-missing.dat diff --git a/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-legacy-with-create-if-missing.dat b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-legacy-with-create-if-missing.dat Binary files differnew file mode 100644 index 00000000000..840bd693670 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-legacy-with-create-if-missing.dat diff --git a/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-no-create-if-missing-uncached.dat b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-no-create-if-missing-uncached.dat Binary files differnew file mode 100644 index 00000000000..f1ceef0e51a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-no-create-if-missing-uncached.dat diff --git a/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-no-create-if-missing.dat b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-no-create-if-missing.dat Binary files differnew file mode 100644 index 00000000000..fc42a504f8b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-no-create-if-missing.dat diff --git a/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-with-create-if-missing-uncached.dat b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-with-create-if-missing-uncached.dat Binary files differnew file mode 100644 index 00000000000..840bd693670 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-with-create-if-missing-uncached.dat diff --git a/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-with-create-if-missing.dat b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-with-create-if-missing.dat Binary files differnew file mode 100644 index 00000000000..ea4852b2e7f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/8.310-cpp-UpdateDocumentMessage-with-create-if-missing.dat diff --git a/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-legacy-no-create-if-missing.dat b/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-legacy-no-create-if-missing.dat Binary files differnew file mode 100644 index 00000000000..f1ceef0e51a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-legacy-no-create-if-missing.dat diff --git a/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-legacy-with-create-if-missing.dat b/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-legacy-with-create-if-missing.dat Binary files differnew file mode 100644 index 00000000000..840bd693670 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-legacy-with-create-if-missing.dat diff --git a/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-no-create-if-missing.dat b/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-no-create-if-missing.dat Binary files differnew file mode 100644 index 00000000000..fc42a504f8b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-no-create-if-missing.dat diff --git a/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-with-create-if-missing.dat b/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-with-create-if-missing.dat Binary files differnew file mode 100644 index 00000000000..ea4852b2e7f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage-with-create-if-missing.dat diff --git a/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage.dat Binary files differindex f1ceef0e51a..fc42a504f8b 100644 --- a/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage.dat +++ b/documentapi/test/crosslanguagefiles/8.310-java-UpdateDocumentMessage.dat diff --git a/eval/src/vespa/eval/eval/gbdt.cpp b/eval/src/vespa/eval/eval/gbdt.cpp index 3422228b03c..7ab4c4ae822 100644 --- a/eval/src/vespa/eval/eval/gbdt.cpp +++ b/eval/src/vespa/eval/eval/gbdt.cpp @@ -154,6 +154,9 @@ Optimize::select_best(const ForestStats &stats, if ((stats.tree_sizes.back().size > 12) && (path_len > 2500.0)) { return apply_chain(VMForest::optimize_chain, stats, trees); } + if (stats.total_size > 25000) { + return apply_chain(VMForest::optimize_chain, stats, trees); + } return Optimize::Result(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index a3fe010c65b..df7720f74f1 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -48,6 +48,13 @@ public class Flags { private static volatile TreeMap<FlagId, FlagDefinition> flags = new TreeMap<>(); + public static final UnboundBooleanFlag ATHENZ_SERVICE_ACCOUNTS = defineFeatureFlag( + "athenz-service-accounts", false, + List.of("hakonhall"), "2024-05-06", "2024-07-06", + "Whether to provision new GCP VM instances with a service account that are independent " + + "of the zone, and aligned with the Athenz service names (configserver and tenant-host).", + "Takes effect when provisioning new VM instances"); + public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, List.of("baldersheim"), "2020-12-02", "2024-12-31", @@ -84,18 +91,6 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); - public static final UnboundBooleanFlag NEW_RESOURCES_FORMULA = defineFeatureFlag( - "new-resources-formula", true, - List.of("hakonhall"), "2024-04-25", "2024-05-25", - "Use an easier to understand formula for calculating the memory and disk resources", - "Takes effect on next deployment of an applications."); - - public static final UnboundBooleanFlag FIX_CONFIG_SERVER_HEAP = defineFeatureFlag( - "fix-config-server-heap", false, - List.of("hakonhall"), "2024-04-23", "2024-05-23", - "Base the calculation of the config server JVM heap size on the amount of memory available to the container.", - "Takes effect on start of config server Podman container"); - public static final UnboundStringFlag RESPONSE_SEQUENCER_TYPE = defineStringFlag( "response-sequencer-type", "ADAPTIVE", List.of("baldersheim"), "2020-12-02", "2024-12-31", @@ -418,13 +413,13 @@ public class Flags { public static UnboundBooleanFlag CALYPSO_ENABLED = defineFeatureFlag( "calypso-enabled", true, - List.of("mortent"), "2024-02-19", "2024-05-01", + List.of("mortent"), "2024-02-19", "2024-08-01", "Whether to enable calypso for host", "Takes effect immediately", HOSTNAME); public static UnboundBooleanFlag ATHENZ_PROVIDER = defineFeatureFlag( "athenz-provider", false, - List.of("mortent"), "2024-02-19", "2024-05-01", + List.of("mortent"), "2024-02-19", "2024-08-01", "Whether to use athenz as node identity provider", "Takes effect on next identity refresh", HOSTNAME); @@ -449,6 +444,12 @@ public class Flags { "Whether logserver container should run otel agent", "Takes effect at redeployment", INSTANCE_ID); + public static UnboundBooleanFlag ENCRYPT_DISK = defineFeatureFlag( + "encrypt-disk", true, + List.of("hmusum"), "2024-04-29", "2024-06-01", + "Whether to encrypt disk when provisioning new hosts", + "Will be read only on boot."); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java index 855430f45fc..7481363b737 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java @@ -12,6 +12,9 @@ import com.yahoo.document.annotation.SpanTrees; import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.language.process.TokenType; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; + +import java.util.OptionalInt; import static com.yahoo.language.LinguisticsCase.toLowerCase; @@ -20,8 +23,19 @@ import static com.yahoo.language.LinguisticsCase.toLowerCase; */ public final class ExactExpression extends Expression { - public ExactExpression() { + private int maxTokenLength; + + private ExactExpression(OptionalInt maxTokenLength) { super(DataType.STRING); + this.maxTokenLength = maxTokenLength.isPresent() ? maxTokenLength.getAsInt() : AnnotatorConfig.getDefaultMaxTokenLength(); + } + + public ExactExpression() { + this(OptionalInt.empty());; + } + + public ExactExpression(int maxTokenLength) { + this(OptionalInt.of(maxTokenLength)); } @Override @@ -36,6 +50,12 @@ public final class ExactExpression extends Expression { String next = toLowerCase(prev); SpanTree tree = output.getSpanTree(SpanTrees.LINGUISTICS); + if (next.length() > maxTokenLength) { + if (tree != null) { + output.removeSpanTree(SpanTrees.LINGUISTICS); + } + return; + } SpanList root; if (tree == null) { root = new SpanList(); @@ -64,8 +84,14 @@ public final class ExactExpression extends Expression { } @Override - public String toString() { - return "exact"; + public String toString() + { + StringBuilder ret = new StringBuilder(); + ret.append("exact"); + if (maxTokenLength != AnnotatorConfig.getDefaultMaxTokenLength()) { + ret.append(" max-token-length:" + maxTokenLength); + } + return ret.toString(); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java index b807ad4cb65..a3c404e50c3 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java @@ -66,9 +66,12 @@ public final class TokenizeExpression extends Expression { if (config.getStemMode() != StemMode.NONE) { ret.append(" stem:\""+config.getStemMode()+"\""); } - if (config.hasNonDefaultMaxTokenLength()) { + if (config.hasNonDefaultMaxTokenizeLength()) { ret.append(" max-length:" + config.getMaxTokenizeLength()); } + if (config.hasNonDefaultMaxTokenLength()) { + ret.append(" max-token-length:" + config.getMaxTokenLength()); + } if (config.hasNonDefaultMaxTermOccurrences()) { ret.append(" max-occurrences:" + config.getMaxTermOccurrences()); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java index 7b6f350d831..6522e284fc8 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java @@ -14,14 +14,17 @@ public class AnnotatorConfig implements Cloneable { private StemMode stemMode; private boolean removeAccents; private int maxTermOccurrences; + private int maxTokenLength; private int maxTokenizeLength; public static final int DEFAULT_MAX_TERM_OCCURRENCES; + private static final int DEFAULT_MAX_TOKEN_LENGTH; private static final int DEFAULT_MAX_TOKENIZE_LENGTH; static { IlscriptsConfig defaults = new IlscriptsConfig(new IlscriptsConfig.Builder()); DEFAULT_MAX_TERM_OCCURRENCES = defaults.maxtermoccurrences(); + DEFAULT_MAX_TOKEN_LENGTH = defaults.maxtokenlength(); DEFAULT_MAX_TOKENIZE_LENGTH = defaults.fieldmatchmaxlength(); } @@ -30,6 +33,7 @@ public class AnnotatorConfig implements Cloneable { stemMode = StemMode.NONE; removeAccents = false; maxTermOccurrences = DEFAULT_MAX_TERM_OCCURRENCES; + maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH; maxTokenizeLength = DEFAULT_MAX_TOKENIZE_LENGTH; } @@ -38,6 +42,7 @@ public class AnnotatorConfig implements Cloneable { stemMode = rhs.stemMode; removeAccents = rhs.removeAccents; maxTermOccurrences = rhs.maxTermOccurrences; + maxTokenLength = rhs.maxTokenLength; maxTokenizeLength = rhs.maxTokenizeLength; } @@ -82,7 +87,18 @@ public class AnnotatorConfig implements Cloneable { return this; } - public AnnotatorConfig setMaxTokenLength(int maxTokenizeLength) { + public AnnotatorConfig setMaxTokenLength(int maxTokenLength) { + this.maxTokenLength = maxTokenLength; + return this; + } + + public int getMaxTokenLength() { + return maxTokenLength; + } + + public static int getDefaultMaxTokenLength() { return DEFAULT_MAX_TOKEN_LENGTH; } + + public AnnotatorConfig setMaxTokenizeLength(int maxTokenizeLength) { this.maxTokenizeLength = maxTokenizeLength; return this; } @@ -92,6 +108,10 @@ public class AnnotatorConfig implements Cloneable { } public boolean hasNonDefaultMaxTokenLength() { + return maxTokenLength != DEFAULT_MAX_TOKEN_LENGTH; + } + + public boolean hasNonDefaultMaxTokenizeLength() { return maxTokenizeLength != DEFAULT_MAX_TOKENIZE_LENGTH; } @@ -116,6 +136,9 @@ public class AnnotatorConfig implements Cloneable { if (maxTermOccurrences != rhs.maxTermOccurrences) { return false; } + if (maxTokenLength != rhs.maxTokenLength) { + return false; + } if (maxTokenizeLength != rhs.maxTokenizeLength) { return false; } @@ -125,7 +148,7 @@ public class AnnotatorConfig implements Cloneable { @Override public int hashCode() { return getClass().hashCode() + language.hashCode() + stemMode.hashCode() + - Boolean.valueOf(removeAccents).hashCode() + maxTermOccurrences + maxTokenizeLength; + Boolean.valueOf(removeAccents).hashCode() + maxTermOccurrences + maxTokenLength + maxTokenizeLength; } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java index 86d4e91a567..913b874c6f6 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java @@ -78,7 +78,8 @@ public class LinguisticsAnnotator { TermOccurrences termOccurrences = new TermOccurrences(config.getMaxTermOccurrences()); SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS); for (Token token : tokens) - addAnnotationSpan(text.getString(), tree.spanList(), token, config.getStemMode(), termOccurrences); + addAnnotationSpan(text.getString(), tree.spanList(), token, config.getStemMode(), termOccurrences, + config.getMaxTokenLength()); if (tree.numAnnotations() == 0) return false; text.setSpanTree(tree); @@ -100,17 +101,22 @@ public class LinguisticsAnnotator { return new Annotation(AnnotationTypes.TERM, new StringFieldValue(term)); } - private static void addAnnotation(Span here, String term, String orig, TermOccurrences termOccurrences) { + private static void addAnnotation(Span here, String term, String orig, TermOccurrences termOccurrences, + int maxTokenLength) { + if (term.length() > maxTokenLength) { + return; + } if (termOccurrences.termCountBelowLimit(term)) { here.annotate(termAnnotation(term, orig)); } } - private static void addAnnotationSpan(String input, SpanList parent, Token token, StemMode mode, TermOccurrences termOccurrences) { + private static void addAnnotationSpan(String input, SpanList parent, Token token, StemMode mode, + TermOccurrences termOccurrences, int maxTokenLength) { if ( ! token.isSpecialToken()) { if (token.getNumComponents() > 0) { for (int i = 0; i < token.getNumComponents(); ++i) { - addAnnotationSpan(input, parent, token.getComponent(i), mode, termOccurrences); + addAnnotationSpan(input, parent, token.getComponent(i), mode, termOccurrences, maxTokenLength); } return; } @@ -130,18 +136,21 @@ public class LinguisticsAnnotator { String lowercasedOrig = toLowerCase(token.getOrig()); String term = token.getTokenString(); if (term != null) { - addAnnotation(where, term, token.getOrig(), termOccurrences); + addAnnotation(where, term, token.getOrig(), termOccurrences, maxTokenLength); if ( ! term.equals(lowercasedOrig)) - addAnnotation(where, lowercasedOrig, token.getOrig(), termOccurrences); + addAnnotation(where, lowercasedOrig, token.getOrig(), termOccurrences, maxTokenLength); } for (int i = 0; i < token.getNumStems(); i++) { String stem = token.getStem(i); if (! (stem.equals(lowercasedOrig) || stem.equals(term))) - addAnnotation(where, stem, token.getOrig(), termOccurrences); + addAnnotation(where, stem, token.getOrig(), termOccurrences, maxTokenLength); } } else { String term = token.getTokenString(); if (term == null || term.trim().isEmpty()) return; + if (term.length() > maxTokenLength) { + return; + } if (termOccurrences.termCountBelowLimit(term)) { parent.span((int)token.getOffset(), token.getOrig().length()).annotate(termAnnotation(term, token.getOrig())); } diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj index 469d96ead60..29ca5270db8 100644 --- a/indexinglanguage/src/main/javacc/IndexingParser.jj +++ b/indexinglanguage/src/main/javacc/IndexingParser.jj @@ -174,6 +174,7 @@ TOKEN : <LOWER_CASE: "lowercase"> | <MAX_LENGTH: "max-length"> | <MAX_OCCURRENCES: "max-occurrences"> | + <MAX_TOKEN_LENGTH: "max-token-length"> | <NGRAM: "ngram"> | <NORMALIZE: "normalize"> | <NOW: "now"> | @@ -407,10 +408,13 @@ Expression embedExp() : { return new EmbedExpression(embedders, embedderId, embedderArguments); } } -Expression exactExp() : { } +Expression exactExp() : { - ( <EXACT> ) - { return new ExactExpression(); } + int maxTokenLength = annotatorCfg.getMaxTokenLength(); +} +{ + ( <EXACT> [ <MAX_TOKEN_LENGTH> <COLON> maxTokenLength = integer() ] ) + { return new ExactExpression(maxTokenLength); } } Expression flattenExp() : { } @@ -686,11 +690,13 @@ AnnotatorConfig tokenizeCfg() : String str = "SHORTEST"; Integer maxLength; Integer maxTermOccurrences; + Integer maxTokenLength; } { ( <STEM> ( <COLON> str = string() ) ? { val.setStemMode(str); } | - <MAX_LENGTH> <COLON> maxLength = integer() { val.setMaxTokenLength(maxLength); } | + <MAX_LENGTH> <COLON> maxLength = integer() { val.setMaxTokenizeLength(maxLength); } | <MAX_OCCURRENCES> <COLON> maxTermOccurrences = integer() { val.setMaxTermOccurrences(maxTermOccurrences); } | + <MAX_TOKEN_LENGTH> <COLON> maxTokenLength = integer() { val.setMaxTokenLength(maxTokenLength); } | <NORMALIZE> { val.setRemoveAccents(true); } )+ { return val; } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java index 403d1820f70..b338c45f7a4 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java @@ -63,6 +63,15 @@ public class ExactTestCase { } @Test + public void requireThatLongStringsAreNotAnnotated() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("foo")); + new ExactExpression(2).execute(ctx); + + assertNull(((StringFieldValue)ctx.getValue()).getSpanTree(SpanTrees.LINGUISTICS)); + } + + @Test public void requireThatEmptyStringsAreNotAnnotated() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); ctx.setValue(new StringFieldValue("")); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java index 01ffbe359f3..7ed3ab410a3 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java @@ -62,4 +62,15 @@ public class TokenizeTestCase { assertTrue(val instanceof StringFieldValue); assertNotNull(((StringFieldValue)val).getSpanTree(SpanTrees.LINGUISTICS)); } + + @Test + public void requireThatLongWordIsDropped() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("foo")); + new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig().setMaxTokenLength(2)).execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof StringFieldValue); + assertNull(((StringFieldValue)val).getSpanTree(SpanTrees.LINGUISTICS)); + } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfigTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfigTestCase.java index 0d34d2841fd..c3131e28906 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfigTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfigTestCase.java @@ -27,6 +27,8 @@ public class AnnotatorConfigTestCase { assertTrue(config.getRemoveAccents()); config.setRemoveAccents(false); assertFalse(config.getRemoveAccents()); + config.setMaxTokenLength(10); + assertEquals(10, config.getMaxTokenLength()); } @Test @@ -35,11 +37,13 @@ public class AnnotatorConfigTestCase { config.setLanguage(Language.ARABIC); config.setStemMode(StemMode.SHORTEST); config.setRemoveAccents(!config.getRemoveAccents()); + config.setMaxTokenLength(11); AnnotatorConfig other = new AnnotatorConfig(config); assertEquals(config.getLanguage(), other.getLanguage()); assertEquals(config.getStemMode(), other.getStemMode()); assertEquals(config.getRemoveAccents(), other.getRemoveAccents()); + assertEquals(config.getMaxTokenLength(), other.getMaxTokenLength()); } @Test @@ -49,6 +53,7 @@ public class AnnotatorConfigTestCase { assertFalse(config.equals(newConfig(Language.SPANISH, StemMode.SHORTEST, false))); assertFalse(config.equals(newConfig(Language.DUTCH, StemMode.SHORTEST, false))); assertFalse(config.equals(newConfig(Language.DUTCH, StemMode.NONE, false))); + assertNotEquals(config, newConfig(Language.DUTCH, StemMode.NONE, true).setMaxTokenLength(10)); assertEquals(config, newConfig(Language.DUTCH, StemMode.NONE, true)); assertEquals(config.hashCode(), newConfig(Language.DUTCH, StemMode.NONE, true).hashCode()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java index 136e71564d8..461c915acef 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java @@ -194,7 +194,7 @@ public class LinguisticsAnnotatorTestCase { Linguistics linguistics = new SimpleLinguistics(); - LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, new AnnotatorConfig().setMaxTokenLength(12)); + LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, new AnnotatorConfig().setMaxTokenizeLength(12)); assertTrue(annotator.annotate(shortValue)); assertEquals(spanTree, shortValue.getSpanTree(SpanTrees.LINGUISTICS)); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java index a7ed7ae3e72..1b7c6973f1e 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java @@ -27,6 +27,7 @@ public class ExpressionTestCase { assertExpression(ClearStateExpression.class, "clear_state"); assertExpression(EchoExpression.class, "echo"); assertExpression(ExactExpression.class, "exact"); + assertExpression(ExactExpression.class, "exact max-token-length: 10", Optional.of("exact max-token-length:10")); assertExpression(FlattenExpression.class, "flatten"); assertExpression(ForEachExpression.class, "for_each { 1 }"); assertExpression(GetFieldExpression.class, "get_field field1"); @@ -73,6 +74,7 @@ public class ExpressionTestCase { assertExpression(TokenizeExpression.class, "tokenize stem:\"ALL\""); assertExpression(TokenizeExpression.class, "tokenize normalize"); assertExpression(TokenizeExpression.class, "tokenize max-occurrences: 15", Optional.of("tokenize max-occurrences:15")); + assertExpression(TokenizeExpression.class, "tokenize max-token-length: 15", Optional.of("tokenize max-token-length:15")); assertExpression(ToLongExpression.class, "to_long"); assertExpression(ToPositionExpression.class, "to_pos"); assertExpression(ToStringExpression.class, "to_string"); diff --git a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java index a5387a87aee..f16ea512a45 100644 --- a/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java @@ -102,6 +102,7 @@ public class SdSyntaxHighlighter extends SyntaxHighlighterBase { keyWords.add(SdTypes.ONNX_MODEL); keyWords.add(SdTypes.ANNOTATION); keyWords.add(SdTypes.RANK_PROFILE); + keyWords.add(SdTypes.SIGNIFICANCE); keyWords.add(SdTypes.MATCH_PHASE); keyWords.add(SdTypes.FIRST_PHASE); keyWords.add(SdTypes.EXPRESSION); diff --git a/integration/intellij/src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex b/integration/intellij/src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex index 47050479633..2e80be34fc2 100644 --- a/integration/intellij/src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex +++ b/integration/intellij/src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex @@ -149,7 +149,8 @@ WORD = \w+ "strict" { return STRICT; } "rank-properties" { return RANK_PROPERTIES; } "inputs" { return INPUTS; } - + + "significance" { return SIGNIFICANCE; } "first-phase" { return FIRST_PHASE; } "keep-rank-count" { return KEEP_RANK_COUNT; } "rank-score-drop-limit" { return RANK_SCORE_DROP_LIMIT; } diff --git a/jdisc_core/src/test/resources/exportPackages.properties b/jdisc_core/src/test/resources/exportPackages.properties index 31bb90216e3..877edc19320 100644 --- a/jdisc_core/src/test/resources/exportPackages.properties +++ b/jdisc_core/src/test/resources/exportPackages.properties @@ -1,3 +1,3 @@ #generated by com.yahoo.jdisc.core.ExportPackages #Fri Jul 07 16:04:11 CEST 2023 -exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", java.util.jar; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.security.auth.callback; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="33.1.0",com.google.common.base;version\="33.1.0";uses\:\="javax.annotation",com.google.common.cache;version\="33.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent,javax.annotation",com.google.common.collect;version\="33.1.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.escape;version\="33.1.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.eventbus;version\="33.1.0",com.google.common.graph;version\="33.1.0";uses\:\="com.google.common.collect,javax.annotation",com.google.common.hash;version\="33.1.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.html;version\="33.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="33.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash,javax.annotation",com.google.common.math;version\="33.1.0";uses\:\="javax.annotation",com.google.common.net;version\="33.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape,javax.annotation",com.google.common.primitives;version\="33.1.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.reflect;version\="33.1.0";uses\:\="com.google.common.collect,com.google.common.io,javax.annotation",com.google.common.util.concurrent;version\="33.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal,javax.annotation",com.google.common.xml;version\="33.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, com.sun.jna;version\=5.11.0, com.sun.jna.ptr;version\=5.11.0, com.sun.jna.win32;version\=5.11.0, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.llm.client.openai;version\=1.0.0,ai.vespa.llm.completion;version\=1.0.0,ai.vespa.llm.test;version\=1.0.0,ai.vespa.llm;version\=1.0.0,ai.vespa.net;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.helpers;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" +exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", java.util.jar; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.security.auth.callback; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="33.2.0",com.google.common.base;version\="33.2.0";uses\:\="javax.annotation",com.google.common.cache;version\="33.2.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent,javax.annotation",com.google.common.collect;version\="33.2.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.escape;version\="33.2.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.eventbus;version\="33.2.0",com.google.common.graph;version\="33.2.0";uses\:\="com.google.common.collect,javax.annotation",com.google.common.hash;version\="33.2.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.html;version\="33.2.0";uses\:\="com.google.common.escape",com.google.common.io;version\="33.2.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash,javax.annotation",com.google.common.math;version\="33.2.0";uses\:\="javax.annotation",com.google.common.net;version\="33.2.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape,javax.annotation",com.google.common.primitives;version\="33.2.0";uses\:\="com.google.common.base,javax.annotation",com.google.common.reflect;version\="33.2.0";uses\:\="com.google.common.collect,com.google.common.io,javax.annotation",com.google.common.util.concurrent;version\="33.2.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal,javax.annotation",com.google.common.xml;version\="33.2.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, com.sun.jna;version\=5.11.0, com.sun.jna.ptr;version\=5.11.0, com.sun.jna.win32;version\=5.11.0, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.llm.client.openai;version\=1.0.0,ai.vespa.llm.completion;version\=1.0.0,ai.vespa.llm.test;version\=1.0.0,ai.vespa.llm;version\=1.0.0,ai.vespa.net;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.helpers;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index 1ca32a2dd37..ceab5025760 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -803,7 +803,8 @@ "abstract" ], "methods" : [ - "public abstract com.yahoo.language.significance.DocumentFrequency documentFrequency(java.lang.String)" + "public abstract com.yahoo.language.significance.DocumentFrequency documentFrequency(java.lang.String)", + "public abstract java.lang.String getId()" ], "fields" : [ ] }, diff --git a/linguistics/src/main/java/com/yahoo/language/significance/SignificanceModel.java b/linguistics/src/main/java/com/yahoo/language/significance/SignificanceModel.java index a9f1e48af62..c8a31e1892c 100644 --- a/linguistics/src/main/java/com/yahoo/language/significance/SignificanceModel.java +++ b/linguistics/src/main/java/com/yahoo/language/significance/SignificanceModel.java @@ -9,4 +9,6 @@ import com.yahoo.api.annotations.Beta; @Beta public interface SignificanceModel { DocumentFrequency documentFrequency(String word); + + String getId(); } diff --git a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModel.java b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModel.java index 7ed6f442610..3244b8373ad 100644 --- a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModel.java +++ b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModel.java @@ -1,13 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.significance.impl; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.language.significance.DocumentFrequency; import com.yahoo.language.significance.SignificanceModel; +import java.io.IOException; import java.nio.file.Path; import java.util.HashMap; @@ -18,70 +16,22 @@ import java.util.HashMap; public class DefaultSignificanceModel implements SignificanceModel { private final long corpusSize; private final HashMap<String, Long> frequencies; - private final Path path; - @JsonIgnoreProperties(ignoreUnknown = true) - public static class SignificanceModelFile { - private final String version; - private final String id; - private final String description; - private final long corpusSize; - private final String language; - - private final long wordCount; - private final HashMap<String, Long> frequencies; - - @JsonCreator - public SignificanceModelFile( - @JsonProperty("version") String version, - @JsonProperty("id") String id, - @JsonProperty("description") String description, - @JsonProperty("corpus-size") long corpusSize, - @JsonProperty("language") String language, - @JsonProperty("word-count") long wordCount, - @JsonProperty("frequencies") HashMap<String, Long> frequencies) { - this.version = version; - this.id = id; - this.description = description; - this.corpusSize = corpusSize; - this.language = language; - this.wordCount = wordCount; - this.frequencies = frequencies; - } - - @JsonProperty("version") - public String version() { return version; } - - @JsonProperty("id") - public String id() { return id; } - - @JsonProperty("description") - public String description() { return description; } - - @JsonProperty("corpus-size") - public long corpusSize() { return corpusSize; } - - @JsonProperty("language") - public String language() { return language; } - - @JsonProperty("frequencies") - public HashMap<String, Long> frequencies() { return frequencies; } - - @JsonProperty("word-count") - public long wordCount() { return wordCount; } + private String id; + public DefaultSignificanceModel(DocumentFrequencyFile file, String id) { + this.frequencies = file.frequencies(); + this.corpusSize = file.documentCount(); + this.id = id; } public DefaultSignificanceModel(Path path) { - this.path = path; - ObjectMapper objectMapper = new ObjectMapper(); - try { - SignificanceModelFile model = objectMapper.readValue(this.path.toFile(), SignificanceModelFile.class); - this.corpusSize = model.corpusSize; - this.frequencies = model.frequencies; - } catch (Exception e) { + var file = objectMapper.readValue(path.toFile(), DocumentFrequencyFile.class); + this.frequencies = file.frequencies(); + this.corpusSize = file.documentCount(); + } catch (IOException e) { throw new RuntimeException("Failed to load model from " + path, e); } } @@ -93,4 +43,10 @@ public class DefaultSignificanceModel implements SignificanceModel { } return new DocumentFrequency(1, corpusSize); } + + @Override + public String getId() { + return this.id; + } + } diff --git a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java index 1be1d3f13b5..72874c15d9e 100644 --- a/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java +++ b/linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java @@ -1,20 +1,21 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.significance.impl; +import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.component.annotation.Inject; import com.yahoo.language.Language; import com.yahoo.language.significance.SignificanceModel; import com.yahoo.language.significance.SignificanceModelRegistry; import com.yahoo.search.significance.config.SignificanceConfig; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.EnumMap; -import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.Supplier; -import static com.yahoo.yolean.Exceptions.uncheck; /** * Default implementation of {@link SignificanceModelRegistry}. * This implementation loads models lazily and caches them. @@ -24,24 +25,35 @@ import static com.yahoo.yolean.Exceptions.uncheck; public class DefaultSignificanceModelRegistry implements SignificanceModelRegistry { private final Map<Language, SignificanceModel> models; + @Inject - public DefaultSignificanceModelRegistry(SignificanceConfig cfg) { this(new Builder(cfg)); } - private DefaultSignificanceModelRegistry(Builder b) { + public DefaultSignificanceModelRegistry(SignificanceConfig cfg) { this.models = new EnumMap<>(Language.class); - b.models.forEach((language, path) -> { - models.put(language, - uncheck(() -> new DefaultSignificanceModel(path))); - }); + for (var model : cfg.model()) { + addModel(model.path()); + } } - public DefaultSignificanceModelRegistry(HashMap<Language, Path> map) { + public DefaultSignificanceModelRegistry(List<Path> models) { this.models = new EnumMap<>(Language.class); - map.forEach((language, path) -> { - models.put(language, - uncheck(() -> new DefaultSignificanceModel(path))); - }); + for (var path : models) { + addModel(path); + } } + public void addModel(Path path) { + ObjectMapper objectMapper = new ObjectMapper(); + try { + SignificanceModelFile file = objectMapper.readValue(path.toFile(), SignificanceModelFile.class); + for (var pair : file.languages().entrySet()) { + this.models.put( + Language.fromLanguageTag(pair.getKey()), + new DefaultSignificanceModel(pair.getValue(), file.id())); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to load model from " + path, e); + } + } @Override public Optional<SignificanceModel> getModel(Language language) { @@ -51,20 +63,4 @@ public class DefaultSignificanceModelRegistry implements SignificanceModelRegist } return Optional.of(models.get(language)); } - - - public static final class Builder { - private final Map<Language, Path> models = new EnumMap<>(Language.class); - - public Builder() {} - public Builder(SignificanceConfig cfg) { - for (var model : cfg.model()) { - addModel(Language.fromLanguageTag(model.language()), model.path()); - } - } - - public Builder addModel(Language lang, Path path) { models.put(lang, path); return this; } - public DefaultSignificanceModelRegistry build() { return new DefaultSignificanceModelRegistry(this); } - } - } diff --git a/linguistics/src/main/java/com/yahoo/language/significance/impl/DocumentFrequencyFile.java b/linguistics/src/main/java/com/yahoo/language/significance/impl/DocumentFrequencyFile.java new file mode 100644 index 00000000000..b62754ac8ad --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/significance/impl/DocumentFrequencyFile.java @@ -0,0 +1,43 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.significance.impl; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.HashMap; + +/** + * + * @author MariusArhaug + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class DocumentFrequencyFile { + private final String description; + + private final int documentCount; + + + private final HashMap<String, Long> frequencies; + + @JsonCreator + public DocumentFrequencyFile( + @JsonProperty("description") String description, + @JsonProperty("document-count") int documentCount, + @JsonProperty("document-frequencies") HashMap<String, Long> frequencies) { + this.description = description; + this.documentCount = documentCount; + this.frequencies = frequencies; + } + + @JsonProperty("description") + public String description() { return description; } + + @JsonProperty("document-count") + public int documentCount() { return documentCount; } + + @JsonProperty("document-frequencies") + public HashMap<String, Long> frequencies() { return frequencies; } +} diff --git a/linguistics/src/main/java/com/yahoo/language/significance/impl/SignificanceModelFile.java b/linguistics/src/main/java/com/yahoo/language/significance/impl/SignificanceModelFile.java new file mode 100644 index 00000000000..902613379f0 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/significance/impl/SignificanceModelFile.java @@ -0,0 +1,48 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.significance.impl; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.HashMap; +import java.util.List; + +/** + * + * @author MariusArhaug + */ + +@JsonIgnoreProperties(ignoreUnknown = true) +public class SignificanceModelFile { + private final String version; + private final String id; + private final String description; + + private final HashMap<String, DocumentFrequencyFile> languages; + + @JsonCreator + public SignificanceModelFile( + @JsonProperty("version") String version, + @JsonProperty("id") String id, + @JsonProperty("description") String description, + @JsonProperty("languages") HashMap<String, DocumentFrequencyFile> languages) { + this.version = version; + this.id = id; + this.description = description; + this.languages = languages; + } + + @JsonProperty("version") + public String version() { return version; } + + @JsonProperty("id") + public String id() { return id; } + + @JsonProperty("description") + public String description() { return description; } + + @JsonProperty("languages") + public HashMap<String, DocumentFrequencyFile> languages() { return languages; } +} diff --git a/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java b/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java index d4849571b5e..e8594885b9e 100644 --- a/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java +++ b/linguistics/src/test/java/com/yahoo/language/significance/DefaultSignificanceModelRegistryTest.java @@ -6,7 +6,8 @@ import com.yahoo.language.significance.impl.DefaultSignificanceModelRegistry; import org.junit.Test; import java.nio.file.Path; -import java.util.HashMap; +import java.util.ArrayList; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -18,10 +19,10 @@ public class DefaultSignificanceModelRegistryTest { @Test public void testDefaultSignificanceModelRegistry() { - HashMap<Language, Path> models = new HashMap<>(); + List<Path> models = new ArrayList<>(); - models.put(Language.ENGLISH, Path.of("src/test/models/en.json")); - models.put(Language.NORWEGIAN_BOKMAL, Path.of("src/test/models/no.json")); + models.add(Path.of("src/test/models/docv1.json")); + models.add(Path.of("src/test/models/docv2.json")); DefaultSignificanceModelRegistry defaultSignificanceModelRegistry = new DefaultSignificanceModelRegistry(models); @@ -39,6 +40,45 @@ public class DefaultSignificanceModelRegistryTest { assertNotNull(englishModel); assertNotNull(norwegianModel); + assertEquals("test::2", englishModel.getId()); + assertEquals("test::2", norwegianModel.getId()); + + assertEquals(4, englishModel.documentFrequency("test").frequency()); + assertEquals(14, englishModel.documentFrequency("test").corpusSize()); + + assertEquals(3, norwegianModel.documentFrequency("nei").frequency()); + assertEquals(20, norwegianModel.documentFrequency("nei").corpusSize()); + + assertEquals(1, norwegianModel.documentFrequency("non-existent-word").frequency()); + assertEquals(20, norwegianModel.documentFrequency("non-existent-word").corpusSize()); + + } + + @Test + public void testDefaultSignificanceModelRegistryInOppsiteOrder() { + + List<Path> models = new ArrayList<>(); + + models.add(Path.of("src/test/models/docv2.json")); + models.add(Path.of("src/test/models/docv1.json")); + + DefaultSignificanceModelRegistry defaultSignificanceModelRegistry = new DefaultSignificanceModelRegistry(models); + + var optionalEnglishModel = defaultSignificanceModelRegistry.getModel(Language.ENGLISH); + var optionalNorwegianModel = defaultSignificanceModelRegistry.getModel(Language.NORWEGIAN_BOKMAL); + + assertTrue(optionalEnglishModel.isPresent()); + assertTrue(optionalNorwegianModel.isPresent()); + + var englishModel = optionalEnglishModel.get(); + var norwegianModel = optionalNorwegianModel.get(); + + assertNotNull(englishModel); + assertNotNull(norwegianModel); + + assertEquals("test::1", englishModel.getId()); + assertEquals("test::2", norwegianModel.getId()); + assertEquals(2, englishModel.documentFrequency("test").frequency()); assertEquals(10, englishModel.documentFrequency("test").corpusSize()); @@ -47,6 +87,5 @@ public class DefaultSignificanceModelRegistryTest { assertEquals(1, norwegianModel.documentFrequency("non-existent-word").frequency()); assertEquals(20, norwegianModel.documentFrequency("non-existent-word").corpusSize()); - } } diff --git a/linguistics/src/test/models/docv1.json b/linguistics/src/test/models/docv1.json new file mode 100644 index 00000000000..04010959a58 --- /dev/null +++ b/linguistics/src/test/models/docv1.json @@ -0,0 +1,18 @@ +{ + "version" : "1.0", + "id" : "test::1", + "description" : "desc", + "languages" : { + "en": { + "description" : "english model", + "document-count" : 10, + "language" : "en", + "document-frequencies" : { + "usa" : 2, + "hello": 3, + "world": 5, + "test": 2 + } + } + } +} diff --git a/linguistics/src/test/models/docv2.json b/linguistics/src/test/models/docv2.json new file mode 100644 index 00000000000..c00d02fb744 --- /dev/null +++ b/linguistics/src/test/models/docv2.json @@ -0,0 +1,31 @@ +{ + "version" : "2.0", + "id" : "test::2", + "description" : "desc", + "languages" : { + "en": { + "description" : "english model", + "document-count" : 14, + "document-frequencies" : { + "usa" : 2, + "hello": 3, + "world": 5, + "test": 4, + "additional": 2 + } + }, + "nb": { + "description" : "norwegian model", + "document-count" : 20, + "document-frequencies" : { + "usa" : 2, + "hello": 10, + "verden": 5, + "test": 2, + "norge": 11, + "ja": 12, + "nei": 3 + } + } + } +} diff --git a/linguistics/src/test/models/en.json b/linguistics/src/test/models/en.json index 50bae5e3451..87b7b2faa08 100644 --- a/linguistics/src/test/models/en.json +++ b/linguistics/src/test/models/en.json @@ -1,11 +1,11 @@ { "version" : "1.0", "id" : "test::1", - "description" : "desc", - "corpus-size" : 10, + "description" : "english model", + "document-count" : 10, "language" : "en", "word-count" : 4, - "frequencies" : { + "document-frequencies" : { "usa" : 2, "hello": 3, "world": 5, diff --git a/linguistics/src/test/models/no.json b/linguistics/src/test/models/no.json deleted file mode 100644 index 5fca8929e74..00000000000 --- a/linguistics/src/test/models/no.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "version" : "1.0", - "id" : "test::2", - "description" : "norsk beskrivelse", - "corpus-size" : 20, - "language" : "nb", - "word-count" : 7, - "frequencies" : { - "usa" : 2, - "hello": 10, - "verden": 5, - "test": 2, - "norge": 11, - "ja": 12, - "nei": 3 - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java index 0e33d7dbf2f..052b8425a45 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java @@ -53,8 +53,8 @@ public class MetricsParser { throw new IOException("Expected start of object, got " + parser.currentToken()); } - for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parser.getCurrentName(); + for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.currentName(); JsonToken token = parser.nextToken(); if (fieldName.equals("metrics")) { parseMetrics(parser, consumer); @@ -67,12 +67,12 @@ public class MetricsParser { } static private Instant parseSnapshot(JsonParser parser) throws IOException { - if (parser.getCurrentToken() != JsonToken.START_OBJECT) { + if (parser.currentToken() != JsonToken.START_OBJECT) { throw new IOException("Expected start of 'snapshot' object, got " + parser.currentToken()); } Instant timestamp = Instant.now(); - for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parser.getCurrentName(); + for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.currentName(); JsonToken token = parser.nextToken(); if (fieldName.equals("to")) { timestamp = Instant.ofEpochSecond(parser.getLongValue()); @@ -88,12 +88,12 @@ public class MetricsParser { // 'metrics' object with 'snapshot' and 'values' arrays static private void parseMetrics(JsonParser parser, Collector consumer) throws IOException { - if (parser.getCurrentToken() != JsonToken.START_OBJECT) { + if (parser.currentToken() != JsonToken.START_OBJECT) { throw new IOException("Expected start of 'metrics' object, got " + parser.currentToken()); } Instant timestamp = Instant.now(); - for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parser.getCurrentName(); + for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.currentName(); JsonToken token = parser.nextToken(); if (fieldName.equals("snapshot")) { timestamp = parseSnapshot(parser); @@ -109,7 +109,7 @@ public class MetricsParser { // 'values' array static private void parseMetricValues(JsonParser parser, Instant timestamp, Collector consumer) throws IOException { - if (parser.getCurrentToken() != JsonToken.START_ARRAY) { + if (parser.currentToken() != JsonToken.START_ARRAY) { throw new IOException("Expected start of 'metrics:values' array, got " + parser.currentToken()); } @@ -126,8 +126,8 @@ public class MetricsParser { String description = ""; Map<DimensionId, String> dim = Map.of(); List<Map.Entry<String, Number>> values = List.of(); - for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parser.getCurrentName(); + for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.currentName(); JsonToken token = parser.nextToken(); switch (fieldName) { case "name" -> name = parser.getText(); @@ -154,8 +154,8 @@ public class MetricsParser { Set<Dimension> dimensions = new HashSet<>(); - for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parser.getCurrentName(); + for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.currentName(); JsonToken token = parser.nextToken(); if (token == JsonToken.VALUE_STRING){ @@ -180,17 +180,16 @@ public class MetricsParser { private static List<Map.Entry<String, Number>> parseValues(JsonParser parser) throws IOException { List<Map.Entry<String, Number>> metrics = new ArrayList<>(); - for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parser.getCurrentName(); + for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String metricName = parser.currentName(); JsonToken token = parser.nextToken(); - String metricName = fieldName; if (token == JsonToken.VALUE_NUMBER_INT) { metrics.add(Map.entry(metricName, parser.getLongValue())); } else if (token == JsonToken.VALUE_NUMBER_FLOAT) { double value = parser.getValueAsDouble(); metrics.add(Map.entry(metricName, value == ZERO_DOUBLE ? ZERO_DOUBLE : value)); } else { - throw new IllegalArgumentException("Value for aggregator '" + fieldName + "' is not a number"); + throw new IllegalArgumentException("Value for aggregator '" + metricName + "' is not a number"); } } return metrics; diff --git a/model-integration/abi-spec.json b/model-integration/abi-spec.json index e7130d9c777..31f2b64d728 100644 --- a/model-integration/abi-spec.json +++ b/model-integration/abi-spec.json @@ -94,6 +94,7 @@ "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder model(com.yahoo.config.ModelReference)", "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder parallelRequests(int)", "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder maxQueueSize(int)", + "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder maxQueueWait(int)", "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder useGpu(boolean)", "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder gpuLayers(int)", "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder threads(int)", @@ -139,6 +140,7 @@ "public java.nio.file.Path model()", "public int parallelRequests()", "public int maxQueueSize()", + "public int maxQueueWait()", "public boolean useGpu()", "public int gpuLayers()", "public int threads()", diff --git a/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java b/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java index aa7c071b93a..b6409b5466d 100644 --- a/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java +++ b/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java @@ -3,6 +3,7 @@ package ai.vespa.llm.clients; import ai.vespa.llm.InferenceParameters; import ai.vespa.llm.LanguageModel; +import ai.vespa.llm.LanguageModelException; import ai.vespa.llm.completion.Completion; import ai.vespa.llm.completion.Prompt; import com.yahoo.component.AbstractComponent; @@ -14,10 +15,14 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.logging.Logger; @@ -29,14 +34,19 @@ import java.util.logging.Logger; public class LocalLLM extends AbstractComponent implements LanguageModel { private final static Logger logger = Logger.getLogger(LocalLLM.class.getName()); + + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + private final LlamaModel model; private final ThreadPoolExecutor executor; + private final long queueTimeoutMilliseconds; private final int contextSize; private final int maxTokens; @Inject public LocalLLM(LlmLocalClientConfig config) { executor = createExecutor(config); + queueTimeoutMilliseconds = config.maxQueueWait(); // Maximum number of tokens to generate - need this since some models can just generate infinitely maxTokens = config.maxTokens(); @@ -74,6 +84,7 @@ public class LocalLLM extends AbstractComponent implements LanguageModel { logger.info("Closing LLM model..."); model.close(); executor.shutdownNow(); + scheduler.shutdownNow(); } @Override @@ -104,22 +115,39 @@ public class LocalLLM extends AbstractComponent implements LanguageModel { // Todo: more options? var completionFuture = new CompletableFuture<Completion.FinishReason>(); + var hasStarted = new AtomicBoolean(false); try { - executor.submit(() -> { + Future<?> future = executor.submit(() -> { + hasStarted.set(true); for (LlamaModel.Output output : model.generate(inferParams)) { consumer.accept(Completion.from(output.text, Completion.FinishReason.none)); } completionFuture.complete(Completion.FinishReason.stop); }); + + if (queueTimeoutMilliseconds > 0) { + scheduler.schedule(() -> { + if ( ! hasStarted.get()) { + future.cancel(false); + String error = rejectedExecutionReason("Rejected completion due to timeout waiting to start"); + completionFuture.completeExceptionally(new LanguageModelException(504, error)); + } + }, queueTimeoutMilliseconds, TimeUnit.MILLISECONDS); + } + } catch (RejectedExecutionException e) { // If we have too many requests (active + any waiting in queue), we reject the completion - int activeCount = executor.getActiveCount(); - int queueSize = executor.getQueue().size(); - String error = String.format("Rejected completion due to too many requests, " + - "%d active, %d in queue", activeCount, queueSize); + String error = rejectedExecutionReason("Rejected completion due to too many requests"); throw new RejectedExecutionException(error); } return completionFuture; } + private String rejectedExecutionReason(String prepend) { + int activeCount = executor.getActiveCount(); + int queueSize = executor.getQueue().size(); + return String.format("%s, %d active, %d in queue", prepend, activeCount, queueSize); + } + + } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java index e1d2f8802a6..6a1e2f2562a 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java @@ -34,9 +34,9 @@ public class LightGBMImporter extends ModelImporter { private boolean probe(File modelFile) { try (JsonParser parser = Jackson.mapper().createParser(modelFile)) { while (parser.nextToken() != null) { - JsonToken token = parser.getCurrentToken(); + JsonToken token = parser.currentToken(); if (token == JsonToken.FIELD_NAME) { - if ("tree_info".equals(parser.getCurrentName())) return true; + if ("tree_info".equals(parser.currentName())) return true; } } return false; diff --git a/model-integration/src/main/resources/configdefinitions/llm-local-client.def b/model-integration/src/main/resources/configdefinitions/llm-local-client.def index 4823a53ec46..6b83ffd0751 100755 --- a/model-integration/src/main/resources/configdefinitions/llm-local-client.def +++ b/model-integration/src/main/resources/configdefinitions/llm-local-client.def @@ -8,7 +8,10 @@ model model parallelRequests int default=1 # Additional number of requests to put in queue for processing before starting to reject new requests -maxQueueSize int default=10 +maxQueueSize int default=100 + +# Max number of milliseoncds to wait in the queue before rejecting a request +maxQueueWait int default=10000 # Use GPU useGpu bool default=true @@ -24,6 +27,6 @@ threads int default=-1 # Context is divided between parallel requests. So for 10 parallel requests, each "slot" gets 1/10 of the context contextSize int default=4096 -# Maximum number of tokens to process in one request - overriden by inference parameters +# Maximum number of tokens to process in one request - overridden by inference parameters maxTokens int default=512 diff --git a/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java b/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java index a3b260f3fb5..4db1140d171 100644 --- a/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java +++ b/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java @@ -2,6 +2,7 @@ package ai.vespa.llm.clients; import ai.vespa.llm.InferenceParameters; +import ai.vespa.llm.LanguageModelException; import ai.vespa.llm.completion.Completion; import ai.vespa.llm.completion.Prompt; import ai.vespa.llm.completion.StringPrompt; @@ -96,7 +97,6 @@ public class LocalLLMTest { try { for (int i = 0; i < promptsToUse; i++) { final var seq = i; - completions.set(seq, new StringBuilder()); futures.set(seq, llm.completeAsync(StringPrompt.from(prompts.get(seq)), defaultOptions(), completion -> { completions.get(seq).append(completion.text()); @@ -122,8 +122,9 @@ public class LocalLLMTest { var prompts = testPrompts(); var promptsToUse = prompts.size(); var parallelRequests = 2; - var additionalQueue = 1; - // 7 should be rejected + var additionalQueue = 100; + var queueWaitTime = 10; + // 8 should be rejected due to queue wait time var futures = new ArrayList<CompletableFuture<Completion.FinishReason>>(Collections.nCopies(promptsToUse, null)); var completions = new ArrayList<StringBuilder>(Collections.nCopies(promptsToUse, null)); @@ -131,10 +132,12 @@ public class LocalLLMTest { var config = new LlmLocalClientConfig.Builder() .parallelRequests(parallelRequests) .maxQueueSize(additionalQueue) + .maxQueueWait(queueWaitTime) .model(ModelReference.valueOf(model)); var llm = new LocalLLM(config.build()); var rejected = new AtomicInteger(0); + var timedOut = new AtomicInteger(0); try { for (int i = 0; i < promptsToUse; i++) { final var seq = i; @@ -143,7 +146,14 @@ public class LocalLLMTest { try { var future = llm.completeAsync(StringPrompt.from(prompts.get(seq)), defaultOptions(), completion -> { completions.get(seq).append(completion.text()); - }).exceptionally(exception -> Completion.FinishReason.error); + }).exceptionally(exception -> { + if (exception instanceof LanguageModelException lme) { + if (lme.code() == 504) { + timedOut.incrementAndGet(); + } + } + return Completion.FinishReason.error; + }); futures.set(seq, future); } catch (RejectedExecutionException e) { rejected.incrementAndGet(); @@ -151,13 +161,14 @@ public class LocalLLMTest { } for (int i = 0; i < promptsToUse; i++) { if (futures.get(i) != null) { - assertNotEquals(futures.get(i).join(), Completion.FinishReason.error); + futures.get(i).join(); } } } finally { llm.deconstruct(); } - assertEquals(7, rejected.get()); + assertEquals(0, rejected.get()); + assertEquals(8, timedOut.get()); } private static InferenceParameters defaultOptions() { diff --git a/parent/pom.xml b/parent/pom.xml index 3299a9fb871..45259e567ca 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -317,7 +317,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.29.0</version> + <version>5.30.0</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> @@ -327,7 +327,7 @@ <dependency> <groupId>org.openrewrite.recipe</groupId> <artifactId>rewrite-testing-frameworks</artifactId> - <version>2.7.0</version> + <version>2.8.0</version> </dependency> </dependencies> </plugin> @@ -1199,7 +1199,7 @@ See pluginManagement of rewrite-maven-plugin for more details --> <groupId>org.openrewrite.recipe</groupId> <artifactId>rewrite-recipe-bom</artifactId> - <version>2.10.0</version> + <version>2.11.0</version> <type>pom</type> <scope>import</scope> </dependency> diff --git a/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java b/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java index 09487506ffe..42b6195549e 100644 --- a/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java +++ b/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java @@ -10,7 +10,6 @@ import java.util.Arrays; /** * Parses predicate queries from JSON. - * * Input JSON is assumed to have the following format: * { * "features": [ @@ -46,7 +45,7 @@ public class PredicateQueryParser { try (JsonParser parser = factory.createParser(json)) { skipToken(parser, JsonToken.START_OBJECT); while (parser.nextToken() != JsonToken.END_OBJECT) { - String fieldName = parser.getCurrentName(); + String fieldName = parser.currentName(); switch (fieldName) { case "features": parseFeatures(parser, JsonParser::getText, featureHandler); @@ -82,7 +81,7 @@ public class PredicateQueryParser { long subqueryBitmap = SubqueryBitmap.DEFAULT_VALUE; // Specifying subquery bitmap is optional. while (parser.nextToken() != JsonToken.END_OBJECT) { - String fieldName = parser.getCurrentName(); + String fieldName = parser.currentName(); skipToken(parser, JsonToken.VALUE_STRING, JsonToken.VALUE_NUMBER_INT); switch (fieldName) { case "k": @@ -100,11 +99,11 @@ public class PredicateQueryParser { } if (key == null) { throw new IllegalArgumentException( - String.format("Feature key is missing! (%s)", parser.getCurrentLocation())); + String.format("Feature key is missing! (%s)", parser.currentLocation())); } if (value == null) { throw new IllegalArgumentException( - String.format("Feature value is missing! (%s)", parser.getCurrentLocation())); + String.format("Feature value is missing! (%s)", parser.currentLocation())); } featureHandler.accept(key, value, subqueryBitmap); } @@ -114,7 +113,7 @@ public class PredicateQueryParser { if (Arrays.stream(expected).noneMatch(e -> e.equals(actual))) { throw new IllegalArgumentException( String.format("Expected a token in %s, got %s (%s).", - Arrays.toString(expected), actual, parser.getTokenLocation())); + Arrays.toString(expected), actual, parser.currentTokenLocation())); } } diff --git a/screwdriver/delete-old-cloudsmith-artifacts.sh b/screwdriver/delete-old-cloudsmith-artifacts.sh index e9e67c23823..552175f7a69 100755 --- a/screwdriver/delete-old-cloudsmith-artifacts.sh +++ b/screwdriver/delete-old-cloudsmith-artifacts.sh @@ -2,7 +2,7 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail -MAX_NUMBER_OF_RELEASES=45 +MAX_NUMBER_OF_RELEASES=40 # Cloudsmith repo rpm --import 'https://dl.cloudsmith.io/public/vespa/open-source-rpms/gpg.0F3DA3C70D35DA7B.key' diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp index fd202c24887..732a95e7c13 100644 --- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp @@ -2,11 +2,19 @@ #include <vespa/searchcore/proton/attribute/attribute_manager_explorer.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> +#include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> +#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> +#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchcore/proton/test/attribute_vectors.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/imported_attribute_vector.h> +#include <vespa/searchlib/attribute/imported_attribute_vector_factory.h> #include <vespa/searchlib/attribute/interlock.h> +#include <vespa/searchlib/attribute/reference_attribute.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> @@ -21,6 +29,12 @@ using namespace proton; using namespace proton::test; using search::AttributeVector; using search::DictionaryConfig; +using search::attribute::BasicType; +using search::attribute::Config; +using search::attribute::ImportedAttributeVector; +using search::attribute::ImportedAttributeVectorFactory; +using search::attribute::ReferenceAttribute; +using search::attribute::test::MockGidToLidMapperFactory; using vespalib::ForegroundThreadExecutor; using vespalib::ISequencedTaskExecutor; using vespalib::SequencedTaskExecutor; @@ -32,6 +46,10 @@ using vespalib::HwInfo; const vespalib::string TEST_DIR = "test_output"; +const vespalib::string ref_name("ref"); +const vespalib::string target_name("f3"); +const vespalib::string imported_name("my_f3"); + namespace { VESPA_THREAD_STACK_TAG(test_executor) } @@ -43,28 +61,13 @@ struct AttributesStateExplorerTest : public ::testing::Test std::unique_ptr<ISequencedTaskExecutor> _attribute_field_writer; ForegroundThreadExecutor _shared; HwInfo _hwInfo; + std::shared_ptr<const IDocumentMetaStoreContext> _parent_dms; + std::shared_ptr<IDocumentMetaStoreContext> _dms; + std::shared_ptr<AttributeManager> _parent_mgr; AttributeManager::SP _mgr; AttributeManagerExplorer _explorer; - AttributesStateExplorerTest() - : _dirHandler(TEST_DIR), - _fileHeaderContext(), - _attribute_field_writer(SequencedTaskExecutor::create(test_executor, 1)), - _shared(), - _hwInfo(), - _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), - _fileHeaderContext, - std::make_shared<search::attribute::Interlock>(), - *_attribute_field_writer, - _shared, - _hwInfo)), - _explorer(_mgr) - { - addAttribute("regular"); - addExtraAttribute("extra"); - add_fast_search_attribute("btree", DictionaryConfig::Type::BTREE); - add_fast_search_attribute("hybrid", DictionaryConfig::Type::BTREE_AND_HASH); - add_fast_search_attribute("hash", DictionaryConfig::Type::HASH); - } + AttributesStateExplorerTest() noexcept; + ~AttributesStateExplorerTest() override; void addAttribute(const vespalib::string &name) { _mgr->addAttribute({name, AttributeUtils::getInt32Config()}, 1); } @@ -84,16 +87,71 @@ struct AttributesStateExplorerTest : public ::testing::Test _explorer.get_child(name)->get_state(inserter, true); return result; } - + void add_reference_attribute() { + search::attribute::Config cfg(BasicType::REFERENCE); + _mgr->addAttribute({ ref_name, cfg }, 1); + auto& ref_attr = dynamic_cast<ReferenceAttribute&>(**_mgr->getAttribute(ref_name)); + ref_attr.setGidToLidMapperFactory(std::make_shared<MockGidToLidMapperFactory>()); + } + std::shared_ptr<ReferenceAttribute> get_reference_attribute() { + return std::dynamic_pointer_cast<ReferenceAttribute>(_mgr->getAttribute(ref_name)->getSP()); + } + void add_imported_attributes() { + auto repo = std::make_unique<ImportedAttributesRepo>(); + auto attr = ImportedAttributeVectorFactory::create(imported_name, + get_reference_attribute(), + _dms, + _parent_mgr->getAttribute(target_name)->getSP(), + _parent_dms, + false); + repo->add(imported_name, attr); + _mgr->setImportedAttributes(std::move(repo)); + } }; +AttributesStateExplorerTest::AttributesStateExplorerTest() noexcept + : _dirHandler(TEST_DIR), + _fileHeaderContext(), + _attribute_field_writer(SequencedTaskExecutor::create(test_executor, 1)), + _shared(), + _hwInfo(), + _parent_dms(std::make_shared<const DocumentMetaStoreContext>(std::make_shared<bucketdb::BucketDBOwner>())), + _dms(), + _parent_mgr(std::make_shared<AttributeManager> + (TEST_DIR, "test.parent.subdb", TuneFileAttributes(), + _fileHeaderContext, + std::make_shared<search::attribute::Interlock>(), + *_attribute_field_writer, + _shared, + _hwInfo)), + _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), + _fileHeaderContext, + std::make_shared<search::attribute::Interlock>(), + *_attribute_field_writer, + _shared, + _hwInfo)), + _explorer(_mgr) +{ + _parent_mgr->addAttribute({target_name, AttributeUtils::getInt32Config()}, 1); + addAttribute("regular"); + addExtraAttribute("extra"); + add_fast_search_attribute("btree", DictionaryConfig::Type::BTREE); + add_fast_search_attribute("hybrid", DictionaryConfig::Type::BTREE_AND_HASH); + add_fast_search_attribute("hash", DictionaryConfig::Type::HASH); + add_reference_attribute(); + add_imported_attributes(); +} + +AttributesStateExplorerTest::~AttributesStateExplorerTest() = default; + + using StringVector = std::vector<vespalib::string>; TEST_F(AttributesStateExplorerTest, require_that_attributes_are_exposed_as_children_names) { StringVector children = _explorer.get_children_names(); std::sort(children.begin(), children.end()); - EXPECT_EQ(StringVector({"btree", "hash", "hybrid", "regular"}), children); + EXPECT_EQ(StringVector({"btree", "hash", "hybrid", "my_f3", "ref", "regular"}), children); } TEST_F(AttributesStateExplorerTest, require_that_attributes_are_explorable) @@ -125,4 +183,12 @@ TEST_F(AttributesStateExplorerTest, require_that_dictionary_memory_usage_is_repo } } +TEST_F(AttributesStateExplorerTest, require_that_imported_attribute_shows_memory_usage) +{ + vespalib::string cache_memory_usage("cacheMemoryUsage"); + auto slime = explore_attribute(imported_name); + EXPECT_LT(0, slime[cache_memory_usage]["allocated"].asLong()); + EXPECT_LT(0, slime[cache_memory_usage]["used"].asLong()); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt index 3717c24650d..c1f10ebd80d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt @@ -34,6 +34,7 @@ vespa_add_library(searchcore_attribute STATIC document_field_retriever.cpp filter_attribute_manager.cpp flushableattribute.cpp + imported_attribute_vector_explorer.cpp imported_attributes_context.cpp imported_attributes_repo.cpp initialized_attributes_result.cpp diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp index 0b48afe4ab8..0797ddfb6cd 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp @@ -3,9 +3,13 @@ #include "attribute_manager_explorer.h" #include "attribute_executor.h" #include "attribute_vector_explorer.h" +#include "imported_attribute_vector_explorer.h" +#include "imported_attributes_repo.h" #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/imported_attribute_vector.h> using search::AttributeVector; +using search::attribute::ImportedAttributeVector; using vespalib::slime::Inserter; namespace proton { @@ -32,6 +36,14 @@ AttributeManagerExplorer::get_children_names() const for (const auto &attr : attributes) { names.push_back(attr->getName()); } + auto imported = _mgr->getImportedAttributes(); + if (imported != nullptr) { + std::vector<std::shared_ptr<ImportedAttributeVector>> i_list; + imported->getAll(i_list); + for (const auto& attr : i_list) { + names.push_back(attr->getName()); + } + } return names; } @@ -39,6 +51,15 @@ std::unique_ptr<vespalib::StateExplorer> AttributeManagerExplorer::get_child(vespalib::stringref name) const { auto guard = _mgr->getAttribute(name); + if (!guard || !guard->getSP()) { + auto imported = _mgr->getImportedAttributes(); + if (imported != nullptr) { + auto& imported_attr = imported->get(name); + if (imported_attr) { + return std::make_unique<ImportedAttributeVectorExplorer>(imported_attr); + } + } + } auto attr = guard ? guard->getSP() : std::shared_ptr<AttributeVector>(); if (attr && _mgr->getWritableAttribute(name) != nullptr) { auto executor = std::make_unique<AttributeExecutor>(_mgr, std::move(attr)); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.cpp new file mode 100644 index 00000000000..dd858b78934 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.cpp @@ -0,0 +1,28 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "imported_attribute_vector_explorer.h" +#include <vespa/searchlib/attribute/imported_attribute_vector.h> +#include <vespa/searchlib/util/state_explorer_utils.h> +#include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/util/memoryusage.h> + +using search::StateExplorerUtils; +using search::attribute::ImportedAttributeVector; +using namespace vespalib::slime; + +namespace proton { + +ImportedAttributeVectorExplorer::ImportedAttributeVectorExplorer(std::shared_ptr<ImportedAttributeVector> attr) + : _attr(std::move(attr)) +{ +} + +void +ImportedAttributeVectorExplorer::get_state(const vespalib::slime::Inserter &inserter, bool) const +{ + Cursor &object = inserter.insertObject(); + auto memory_usage = _attr->get_memory_usage(); + StateExplorerUtils::memory_usage_to_slime(memory_usage, object.setObject("cacheMemoryUsage")); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.h b/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.h new file mode 100644 index 00000000000..ce8854b03d2 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.h @@ -0,0 +1,26 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/net/http/state_explorer.h> + +namespace search::attribute { class ImportedAttributeVector; } + +namespace proton { + +/** + * Class used to explore the state of an imported attribute vector. + */ +class ImportedAttributeVectorExplorer : public vespalib::StateExplorer +{ +private: + std::shared_ptr<search::attribute::ImportedAttributeVector> _attr; + +public: + ImportedAttributeVectorExplorer(std::shared_ptr<search::attribute::ImportedAttributeVector> attr); + + // Implements vespalib::StateExplorer + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 0c986422be6..758d1336399 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -210,7 +210,8 @@ private: } FlowStats calculate_flow_stats(uint32_t docid_limit) const override { double rel_est = abs_to_rel_est(_activeLids.size(), docid_limit); - return {rel_est, bitvector_cost(), bitvector_strict_cost(rel_est)}; + double do_not_make_me_strict = 1000.0; + return {rel_est, bitvector_cost(), do_not_make_me_strict * bitvector_strict_cost(rel_est)}; } SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda) const override diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp index 0b2660824c0..919309c5dae 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/queryeval/intermediate_blueprints.h> #include <vespa/searchlib/queryeval/equiv_blueprint.h> #include <vespa/searchlib/queryeval/get_weight_from_node.h> +#include <vespa/searchlib/attribute/attribute_blueprint_params.h> #include <vespa/vespalib/util/issue.h> using namespace search::queryeval; @@ -21,7 +22,7 @@ namespace { struct Mixer { std::unique_ptr<OrBlueprint> attributes; - Mixer() : attributes() {} + Mixer() noexcept: attributes() {} void addAttribute(Blueprint::UP attr) { if ( ! attributes) { @@ -66,7 +67,7 @@ private: void buildIntermediate(IntermediateBlueprint *b, NodeType &n) __attribute__((noinline)); void buildWeakAnd(ProtonWeakAnd &n) { - auto *wand = new WeakAndBlueprint(n.getTargetNumHits()); + auto *wand = new WeakAndBlueprint(n.getTargetNumHits(), _requestContext.get_attribute_blueprint_params().weakand_range); Blueprint::UP result(wand); for (auto node : n.getChildren()) { uint32_t weight = getWeightFromNode(*node).percent(); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index 532ec2f63bd..06290386a31 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -340,6 +340,7 @@ MatchToolsFactory::extract_attribute_blueprint_params(const RankSetup& rank_setu double upper_limit = GlobalFilterUpperLimit::lookup(rank_properties, rank_setup.get_global_filter_upper_limit()); double target_hits_max_adjustment_factor = TargetHitsMaxAdjustmentFactor::lookup(rank_properties, rank_setup.get_target_hits_max_adjustment_factor()); auto fuzzy_matching_algorithm = FuzzyAlgorithm::lookup(rank_properties, rank_setup.get_fuzzy_matching_algorithm()); + double weakand_range = temporary::WeakAndRange::lookup(rank_properties, rank_setup.get_weakand_range()); // Note that we count the reserved docid 0 as active. // This ensures that when searchable-copies=1, the ratio is 1.0. @@ -348,7 +349,8 @@ MatchToolsFactory::extract_attribute_blueprint_params(const RankSetup& rank_setu return {lower_limit * active_hit_ratio, upper_limit * active_hit_ratio, target_hits_max_adjustment_factor, - fuzzy_matching_algorithm}; + fuzzy_matching_algorithm, + weakand_range}; } AttributeOperationTask::AttributeOperationTask(const RequestContext & requestContext, diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index b9794bf6a75..d4ae635d760 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -241,10 +241,16 @@ RPCHooksBase::getProtonStatus(FRT_RPCRequest *req) } void -RPCHooksBase::rpc_die(FRT_RPCRequest *) +RPCHooksBase::rpc_die(FRT_RPCRequest * req) { LOG(debug, "RPCHooksBase::rpc_die"); - _exit(0); + req->Detach(); + letProtonDo(makeLambdaTask([req]() { + LOG(debug, "Nap for 10ms and then quickly exit."); + req->Return(); + std::this_thread::sleep_for(10ms); + std::quick_exit(0); + })); } void diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index b1b2235165f..cce72837dad 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -210,8 +210,10 @@ public: } void expect_entry(uint32_t exp_docid, const DoubleVector& exp_vector, const EntryVector& entries) const { EXPECT_EQUAL(1u, entries.size()); - EXPECT_EQUAL(exp_docid, entries.back().first); - EXPECT_EQUAL(exp_vector, entries.back().second); + if (entries.size() >= 1u) { + EXPECT_EQUAL(exp_docid, entries.back().first); + EXPECT_EQUAL(exp_vector, entries.back().second); + } } void expect_add(uint32_t exp_docid, const DoubleVector& exp_vector) const { expect_entry(exp_docid, exp_vector, _adds); @@ -329,6 +331,10 @@ public: static search::tensor::DistanceFunctionFactory::UP my_dist_fun = search::tensor::make_distance_function_factory(search::attribute::DistanceMetric::Euclidean, vespalib::eval::CellType::DOUBLE); return *my_dist_fun; } + + uint32_t check_consistency(uint32_t) const noexcept override { + return 0; + } }; class MockNearestNeighborIndexFactory : public NearestNeighborIndexFactory { @@ -1077,8 +1083,22 @@ TEST_F("Populates address space usage in mixed tensor attribute with hnsw index" class DenseTensorAttributeMockIndex : public Fixture { public: DenseTensorAttributeMockIndex() : Fixture(vec_2d_spec, FixtureTraits().mock_hnsw()) {} + void add_vec_a(); }; +void +DenseTensorAttributeMockIndex::add_vec_a() +{ + auto& index = mock_index(); + auto vec_a = vec_2d(3, 5); + auto prepare_result = prepare_set_tensor(1, vec_a); + index.expect_prepare_add(1, {3, 5}); + complete_set_tensor(1, vec_a, std::move(prepare_result)); + assertGetTensor(vec_a, 1); + index.expect_complete_add(1, {3, 5}); + index.clear(); +} + TEST_F("setTensor() updates nearest neighbor index", DenseTensorAttributeMockIndex) { auto& index = f.mock_index(); @@ -1097,15 +1117,7 @@ TEST_F("setTensor() updates nearest neighbor index", DenseTensorAttributeMockInd TEST_F("nearest neighbor index can be updated in two phases", DenseTensorAttributeMockIndex) { auto& index = f.mock_index(); - { - auto vec_a = vec_2d(3, 5); - auto prepare_result = f.prepare_set_tensor(1, vec_a); - index.expect_prepare_add(1, {3, 5}); - f.complete_set_tensor(1, vec_a, std::move(prepare_result)); - f.assertGetTensor(vec_a, 1); - index.expect_complete_add(1, {3, 5}); - } - index.clear(); + f.add_vec_a(); { // Replaces previous value. auto vec_b = vec_2d(7, 9); @@ -1121,15 +1133,7 @@ TEST_F("nearest neighbor index can be updated in two phases", DenseTensorAttribu TEST_F("nearest neighbor index is NOT updated when tensor value is unchanged", DenseTensorAttributeMockIndex) { auto& index = f.mock_index(); - { - auto vec_a = vec_2d(3, 5); - auto prepare_result = f.prepare_set_tensor(1, vec_a); - index.expect_prepare_add(1, {3, 5}); - f.complete_set_tensor(1, vec_a, std::move(prepare_result)); - f.assertGetTensor(vec_a, 1); - index.expect_complete_add(1, {3, 5}); - } - index.clear(); + f.add_vec_a(); { // Replaces previous value with the same value auto vec_b = vec_2d(3, 5); @@ -1139,6 +1143,39 @@ TEST_F("nearest neighbor index is NOT updated when tensor value is unchanged", D f.complete_set_tensor(1, vec_b, std::move(prepare_result)); f.assertGetTensor(vec_b, 1); index.expect_empty_complete_add(); + index.expect_empty_add(); + } +} + +TEST_F("nearest neighbor index is updated when value changes from A to B to A", DenseTensorAttributeMockIndex) +{ + auto& index = f.mock_index(); + f.add_vec_a(); + { + // Prepare replace of A with B + auto vec_b = vec_2d(7, 9); + auto prepare_result_b = f.prepare_set_tensor(1, vec_b); + index.expect_prepare_add(1, {7, 9}); + index.clear(); + // Prepare replace of B with A, but prepare sees original A + auto vec_a = vec_2d(3, 5); + auto prepare_result_a = f.prepare_set_tensor(1, vec_a); + EXPECT_TRUE(prepare_result_a.get() == nullptr); + index.expect_empty_prepare_add(); + index.clear(); + // Complete set B + f.complete_set_tensor(1, vec_b, std::move(prepare_result_b)); + index.expect_remove(1, {3, 5}); + f.assertGetTensor(vec_b, 1); + index.expect_complete_add(1, {7, 9}); + index.expect_empty_add(); + index.clear(); + // Complete set A, no prepare result but tensor cells changed + f.complete_set_tensor(1, vec_a, std::move(prepare_result_a)); + index.expect_remove(1, {7, 9}); + index.expect_empty_complete_add(); + index.expect_add(1, {3, 5}); + f.assertGetTensor(vec_a, 1); } } diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index bddc9f92111..490f221d1d8 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -27,8 +27,9 @@ LOG_SETUP("blueprint_test"); using namespace search::queryeval; -using namespace search::fef; using namespace search::query; +using search::fef::MatchData; +using search::queryeval::Blueprint; using search::BitVector; using BlueprintVector = std::vector<std::unique_ptr<Blueprint>>; using vespalib::Slime; @@ -575,7 +576,9 @@ void compare(const Blueprint &bp1, const Blueprint &bp2, bool expect_eq) { bp1.asSlime(SlimeInserter(a)); bp2.asSlime(SlimeInserter(b)); if (expect_eq) { - EXPECT_TRUE(vespalib::slime::are_equal(a.get(), b.get(), cmp_hook)); + if(!EXPECT_TRUE(vespalib::slime::are_equal(a.get(), b.get(), cmp_hook))) { + fprintf(stderr, "a: %s\n\nb: %s\n\n", bp1.asString().c_str(), bp2.asString().c_str()); + } } else { EXPECT_FALSE(vespalib::slime::are_equal(a.get(), b.get(), cmp_hook)); } @@ -613,7 +616,6 @@ TEST_F("test SourceBlender below AND partial optimization", SourceBlenderTestFix auto expect = std::make_unique<AndBlueprint>(); addLeafs(*expect, {1,2,3}); - expect->addChild(addLeafsWithSourceId(std::make_unique<SourceBlenderBlueprint>(f.selector_2), {{10, 1}, {20, 2}})); auto blender = std::make_unique<SourceBlenderBlueprint>(f.selector_1); blender->addChild(addLeafsWithSourceId(3, std::make_unique<AndBlueprint>(), {{30, 3}, {300, 3}})); @@ -621,6 +623,8 @@ TEST_F("test SourceBlender below AND partial optimization", SourceBlenderTestFix blender->addChild(addLeafsWithSourceId(1, std::make_unique<AndBlueprint>(), {{10, 1}, {100, 1}, {1000, 1}})); expect->addChild(std::move(blender)); + expect->addChild(addLeafsWithSourceId(std::make_unique<SourceBlenderBlueprint>(f.selector_2), {{10, 1}, {20, 2}})); + optimize_and_compare(std::move(top), std::move(expect)); } @@ -1401,7 +1405,7 @@ TEST("cost for ANDNOT") { TEST("cost for SB") { InvalidSelector sel; - verify_cost(make::SB(sel), 1.3, 1.3); // max + verify_cost(make::SB(sel), 1.3+1.0, 1.3+(1.0-0.8*0.7*0.5)); // max, non_strict+1.0, strict+est } TEST("cost for NEAR") { diff --git a/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp b/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp index 57fddb0a819..d6008136d73 100644 --- a/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp +++ b/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/queryeval/flow.h> +#include <vespa/searchlib/queryeval/flow_tuning.h> #include <vespa/vespalib/gtest/gtest.h> #include <vector> #include <random> @@ -349,6 +350,35 @@ TEST(FlowTest, blender_flow_cost_accumulation_is_max) { } } +double my_non_strict_cost(double est, double adjust) { + return (1.0/adjust) * flow::forced_strict_cost(FlowStats(est, 0.0, est), adjust); +} + +TEST(FlowTest, non_strict_btree_cost) { + for (double est: {0.001, 0.01, 0.1, 0.2, 0.3, 0.5, 0.75, 1.0}) { + auto prev = FlowStats(est, 1.0, est); + auto base = FlowStats(est, flow::non_strict_cost_of_strict_iterator(est, est), est); + auto opt05 = FlowStats(est, my_non_strict_cost(est, 0.5), est); + auto opt02 = FlowStats(est, my_non_strict_cost(est, 0.2), est); + auto opt01 = FlowStats(est, my_non_strict_cost(est, 0.1), est); + auto opt005 = FlowStats(est, my_non_strict_cost(est, 0.05), est); + auto opt003 = FlowStats(est, my_non_strict_cost(est, 0.03), est); + EXPECT_NEAR(strict_crossover(opt05), 0.5, 1e-6); + EXPECT_NEAR(strict_crossover(opt02), 0.2, 1e-6); + EXPECT_NEAR(strict_crossover(opt01), 0.1, 1e-6); + EXPECT_NEAR(strict_crossover(opt005), 0.05, 1e-6); + EXPECT_NEAR(strict_crossover(opt003), 0.03, 1e-6); + fprintf(stderr, "est: %5.3f\n", est); + fprintf(stderr, " prev crossover: %6.4f (cost: %6.4f)\n", strict_crossover(prev), prev.cost); + fprintf(stderr, " base crossover: %6.4f (cost: %6.4f)\n", strict_crossover(base), base.cost); + fprintf(stderr, " 0.5 crossover: %6.4f (cost: %6.4f)\n", strict_crossover(opt05), opt05.cost); + fprintf(stderr, " 0.2 crossover: %6.4f (cost: %6.4f)\n", strict_crossover(opt02), opt02.cost); + fprintf(stderr, " 0.1 crossover: %6.4f (cost: %6.4f)\n", strict_crossover(opt01), opt01.cost); + fprintf(stderr, " 0.05 crossover: %6.4f (cost: %6.4f)\n", strict_crossover(opt005), opt005.cost); + fprintf(stderr, " 0.03 crossover: %6.4f (cost: %6.4f)\n", strict_crossover(opt003), opt003.cost); + } +} + TEST(FlowTest, optimal_and_flow) { for (size_t i = 0; i < loop_cnt; ++i) { for (bool strict: {false, true}) { diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.cpp b/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.cpp index 8591ec1415d..51177850155 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.cpp +++ b/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.cpp @@ -2,14 +2,14 @@ #include "intermediate_blueprint_factory.h" #include <vespa/searchlib/queryeval/intermediate_blueprints.h> +#include <vespa/searchlib/attribute/singlenumericattribute.h> #include <iomanip> #include <sstream> namespace search::queryeval::test { -template <typename BlueprintType> char -IntermediateBlueprintFactory<BlueprintType>::child_name(void* blueprint) const +IntermediateBlueprintFactory::child_name(void* blueprint) const { auto itr = _child_names.find(blueprint); if (itr != _child_names.end()) { @@ -18,35 +18,33 @@ IntermediateBlueprintFactory<BlueprintType>::child_name(void* blueprint) const return '?'; } -template <typename BlueprintType> -IntermediateBlueprintFactory<BlueprintType>::IntermediateBlueprintFactory(vespalib::stringref name) +IntermediateBlueprintFactory::IntermediateBlueprintFactory(vespalib::stringref name) : _name(name), _children(), _child_names() { } -template <typename BlueprintType> -IntermediateBlueprintFactory<BlueprintType>::~IntermediateBlueprintFactory() = default; +IntermediateBlueprintFactory::~IntermediateBlueprintFactory() = default; -template <typename BlueprintType> std::unique_ptr<Blueprint> -IntermediateBlueprintFactory<BlueprintType>::make_blueprint() +IntermediateBlueprintFactory::make_blueprint() { - auto res = std::make_unique<BlueprintType>(); + auto res = make_self(); _child_names.clear(); char name = 'A'; + uint32_t source = 1; for (const auto& factory : _children) { auto child = factory->make_blueprint(); _child_names[child.get()] = name++; + child->setSourceId(source++); // ignored by non-source-blender blueprints res->addChild(std::move(child)); } return res; } -template <typename BlueprintType> vespalib::string -IntermediateBlueprintFactory<BlueprintType>::get_name(Blueprint& blueprint) const +IntermediateBlueprintFactory::get_name(Blueprint& blueprint) const { auto* intermediate = blueprint.asIntermediate(); if (intermediate != nullptr) { @@ -69,11 +67,29 @@ IntermediateBlueprintFactory<BlueprintType>::get_name(Blueprint& blueprint) cons return get_class_name(blueprint); } -template class IntermediateBlueprintFactory<AndBlueprint>; +//----------------------------------------------------------------------------- AndBlueprintFactory::AndBlueprintFactory() - : IntermediateBlueprintFactory<AndBlueprint>("AND") + : IntermediateBlueprintFactory("AND") {} +std::unique_ptr<IntermediateBlueprint> +AndBlueprintFactory::make_self() const +{ + return std::make_unique<AndBlueprint>(); +} + +//----------------------------------------------------------------------------- + +SourceBlenderBlueprintFactory::SourceBlenderBlueprintFactory() + : IntermediateBlueprintFactory("SB"), + _selector(250, "my_source_blender", 1000) +{} + +std::unique_ptr<IntermediateBlueprint> +SourceBlenderBlueprintFactory::make_self() const +{ + return std::make_unique<SourceBlenderBlueprint>(_selector); } +} diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.h b/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.h index 6f7fe4f9ee7..c791d866612 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.h +++ b/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.h @@ -4,6 +4,7 @@ #include "benchmark_blueprint_factory.h" #include <vespa/searchlib/queryeval/intermediate_blueprints.h> +#include <vespa/searchlib/attribute/fixedsourceselector.h> #include <unordered_map> namespace search::queryeval::test { @@ -11,7 +12,6 @@ namespace search::queryeval::test { /** * Factory that creates an IntermediateBlueprint (of the given type) with children created by the given factories. */ -template <typename BlueprintType> class IntermediateBlueprintFactory : public BenchmarkBlueprintFactory { private: vespalib::string _name; @@ -19,7 +19,8 @@ private: std::unordered_map<void*, char> _child_names; char child_name(void* blueprint) const; - +protected: + virtual std::unique_ptr<IntermediateBlueprint> make_self() const = 0; public: IntermediateBlueprintFactory(vespalib::stringref name); ~IntermediateBlueprintFactory(); @@ -30,10 +31,26 @@ public: vespalib::string get_name(Blueprint& blueprint) const override; }; -class AndBlueprintFactory : public IntermediateBlueprintFactory<AndBlueprint> { +class AndBlueprintFactory : public IntermediateBlueprintFactory { +protected: + std::unique_ptr<IntermediateBlueprint> make_self() const override; public: AndBlueprintFactory(); }; -} +class SourceBlenderBlueprintFactory : public IntermediateBlueprintFactory +{ +private: + FixedSourceSelector _selector; +protected: + std::unique_ptr<IntermediateBlueprint> make_self() const override; +public: + SourceBlenderBlueprintFactory(); + void init_selector(auto f, uint32_t limit) { + for (uint32_t i = 0; i < limit; ++i) { + _selector.setSource(i, f(i)); + } + } +}; +} diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp b/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp index f4a1ade8a66..96472200952 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp +++ b/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp @@ -236,7 +236,8 @@ strict_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit, Planning timer.after(); } FlowStats flow(ctx.blueprint->estimate(), ctx.blueprint->cost(), ctx.blueprint->strict_cost()); - return {timer.min_time() * 1000.0, hits + 1, hits, flow, flow.strict_cost, get_class_name(*ctx.iterator), factory.get_name(*ctx.blueprint)}; + double actual_cost = ctx.blueprint->estimate_actual_cost(InFlow(true)); + return {timer.min_time() * 1000.0, hits + 1, hits, flow, actual_cost, get_class_name(*ctx.iterator), factory.get_name(*ctx.blueprint)}; } template <bool do_unpack> @@ -269,7 +270,7 @@ non_strict_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit, doub timer.after(); } FlowStats flow(ctx.blueprint->estimate(), ctx.blueprint->cost(), ctx.blueprint->strict_cost()); - double actual_cost = flow.cost * filter_hit_ratio; + double actual_cost = ctx.blueprint->estimate_actual_cost(InFlow(filter_hit_ratio)); return {timer.min_time() * 1000.0, seeks, hits, flow, actual_cost, get_class_name(*ctx.iterator), factory.get_name(*ctx.blueprint)}; } @@ -291,10 +292,6 @@ benchmark_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit, bool } } - - - - //----------------------------------------------------------------------------- double est_forced_strict_cost(double estimate, double strict_cost, double rate) { @@ -317,26 +314,26 @@ struct Sample { } }; -double find_crossover(const char *type, const auto &calculate_at, double delta) { +double find_crossover(const char *type, const char *a, const char *b, const auto &calculate_at, double delta) { double min = delta; double max = 1.0; fprintf(stderr, "looking for %s crossover in the range [%g, %g]...\n", type, min, max); auto at_min = calculate_at(min); auto at_max = calculate_at(max); - fprintf(stderr, " before: [%s, %s], after: [%s, %s]\n", - at_min.first.str().c_str(), at_max.first.str().c_str(), - at_min.second.str().c_str(), at_max.second.str().c_str()); - auto best_before = [](auto values) { return (values.first < values.second); }; - if (best_before(at_min) == best_before(at_max)) { + fprintf(stderr, " %s: [%s, %s], %s: [%s, %s]\n", + a, at_min.first.str().c_str(), at_max.first.str().c_str(), + b, at_min.second.str().c_str(), at_max.second.str().c_str()); + auto a_best = [](auto values) { return (values.first < values.second); }; + if (a_best(at_min) == a_best(at_max)) { fprintf(stderr, " NO %s CROSSOVER FOUND\n", type); return 0.0; } while (max > (min + delta)) { double x = (min + max) / 2.0; auto at_x = calculate_at(x); - fprintf(stderr, " best@%g: %s (%s vs %s)\n", x, best_before(at_x) ? "before" : "after", + fprintf(stderr, " best@%g: %s (%s vs %s)\n", x, a_best(at_x) ? a : b, at_x.first.str().c_str(), at_x.second.str().c_str()); - if (best_before(at_min) == best_before(at_x)) { + if (a_best(at_min) == a_best(at_x)) { min = x; at_min = at_x; } else { @@ -409,11 +406,11 @@ void analyze_crossover(BenchmarkBlueprintFactory &fixed, std::function<std::uniq std::vector<double> results; std::vector<const char *> names; names.push_back("time crossover"); - results.push_back(find_crossover("TIME", combine(estimate_AND_time_ms), delta)); + results.push_back(find_crossover("TIME", "before", "after", combine(estimate_AND_time_ms), delta)); names.push_back("cost crossover"); - results.push_back(find_crossover("COST", combine(calculate_AND_cost), delta)); + results.push_back(find_crossover("COST", "before", "after", combine(calculate_AND_cost), delta)); names.push_back("abs_est crossover"); - results.push_back(find_crossover("ABS_EST", combine(first_abs_est), delta)); + results.push_back(find_crossover("ABS_EST", "before", "after", combine(first_abs_est), delta)); sample_at("COST", combine(calculate_AND_cost), results, names); sample_at("TIME", combine(estimate_AND_time_ms), results, names); } @@ -429,21 +426,37 @@ to_string(bool val) void print_result_header() { - std::cout << "| chn | f_ratio | o_ratio | a_ratio | f.est | f.cost | f.scost | hits | seeks | time_ms | act_cost | ns_per_seek | ms_per_act_cost | iterator | blueprint |" << std::endl; + std::cout << "| in_flow | chn | o_ratio | a_ratio | f.est | f.cost | f.act_cost | f.scost | f.act_scost | hits | seeks | time_ms | act_cost | ns_per_seek | ms_per_act_cost | iterator | blueprint |" << std::endl; +} + +std::ostream &operator<<(std::ostream &dst, InFlow in_flow) { + auto old_w = dst.width(); + auto old_p = dst.precision(); + dst << std::setw(7) << std::setprecision(5); + if (in_flow.strict()) { + dst << " STRICT"; + } else { + dst << in_flow.rate(); + } + dst << std::setw(old_w); + dst << std::setprecision(old_p); + return dst; } void -print_result(const BenchmarkResult& res, uint32_t children, double op_hit_ratio, double filter_hit_ratio, uint32_t num_docs) +print_result(const BenchmarkResult& res, uint32_t children, double op_hit_ratio, InFlow in_flow, uint32_t num_docs) { std::cout << std::fixed << std::setprecision(5) - << "| " << std::setw(5) << children - << " | " << std::setw(7) << filter_hit_ratio + << "| " << in_flow + << " | " << std::setw(5) << children << " | " << std::setw(7) << op_hit_ratio << " | " << std::setw(7) << ((double) res.hits / (double) num_docs) << " | " << std::setw(6) << res.flow.estimate << std::setprecision(4) << " | " << std::setw(9) << res.flow.cost + << " | " << std::setw(10) << (res.flow.cost * in_flow.rate()) << " | " << std::setw(7) << res.flow.strict_cost + << " | " << std::setw(11) << (in_flow.strict() ? res.flow.strict_cost : flow::forced_strict_cost(res.flow, in_flow.rate())) << " | " << std::setw(8) << res.hits << " | " << std::setw(8) << res.seeks << std::setprecision(3) @@ -640,7 +653,7 @@ run_benchmark_case(const BenchmarkCaseSetup& setup) if (filter_hit_ratio * setup.filter_crossover_factor <= op_hit_ratio) { auto res = benchmark_search(*factory, setup.num_docs + 1, setup.bcase.strict_context, setup.bcase.force_strict, setup.bcase.unpack_iterator, filter_hit_ratio, PlanningAlgo::Cost); - print_result(res, children, op_hit_ratio, filter_hit_ratio, setup.num_docs); + print_result(res, children, op_hit_ratio, InFlow(setup.bcase.strict_context, filter_hit_ratio), setup.num_docs); result.add(res); } } @@ -681,23 +694,25 @@ run_benchmarks(const BenchmarkSetup& setup) void print_intermediate_blueprint_result_header(size_t children) { + std::cout << "| in_flow"; // This matches the naming scheme in IntermediateBlueprintFactory. char name = 'A'; for (size_t i = 0; i < children; ++i) { - std::cout << "| " << name++ << ".ratio "; + std::cout << " | " << name++ << ".ratio"; } - std::cout << "| flow.cost | flow.scost | flow.est | ratio | hits | seeks | ms_per_cost | time_ms | algo | blueprint |" << std::endl; + std::cout << " | flow.cost | flow.scost | flow.est | ratio | hits | seeks | ms_per_cost | time_ms | algo | blueprint |" << std::endl; } void -print_intermediate_blueprint_result(const BenchmarkResult& res, const std::vector<double>& children_ratios, PlanningAlgo algo, uint32_t num_docs) +print_intermediate_blueprint_result(const BenchmarkResult& res, const std::vector<double>& children_ratios, PlanningAlgo algo, InFlow in_flow, uint32_t num_docs) { - std::cout << std::fixed << std::setprecision(5); + std::cout << std::fixed << std::setprecision(5) + << "| " << in_flow; for (auto ratio : children_ratios) { - std::cout << "| " << std::setw(7) << ratio << " "; + std::cout << " | " << std::setw(7) << ratio; } std::cout << std::setprecision(5) - << "| " << std::setw(10) << res.flow.cost + << " | " << std::setw(10) << res.flow.cost << " | " << std::setw(10) << res.flow.strict_cost << " | " << std::setw(8) << res.flow.estimate << " | " << std::setw(7) << ((double) res.hits / (double) num_docs) @@ -745,9 +760,8 @@ struct BlueprintFactorySetup { BlueprintFactorySetup::~BlueprintFactorySetup() = default; -template <typename IntermediateBlueprintFactoryType> void -run_intermediate_blueprint_benchmark(const BlueprintFactorySetup& a, const BlueprintFactorySetup& b, size_t num_docs) +run_intermediate_blueprint_benchmark(auto factory_factory, std::vector<InFlow> in_flows, const BlueprintFactorySetup& a, const BlueprintFactorySetup& b, size_t num_docs) { print_intermediate_blueprint_result_header(2); double max_speedup = 0.0; @@ -755,26 +769,28 @@ run_intermediate_blueprint_benchmark(const BlueprintFactorySetup& a, const Bluep for (double b_hit_ratio: b.op_hit_ratios) { auto b_factory = b.make_factory_shared(num_docs, b_hit_ratio); for (double a_hit_ratio : a.op_hit_ratios) { - IntermediateBlueprintFactoryType factory; - factory.add_child(a.make_factory(num_docs, a_hit_ratio)); - factory.add_child(b_factory); + auto factory = factory_factory(); + factory->add_child(a.make_factory(num_docs, a_hit_ratio)); + factory->add_child(b_factory); double time_ms_esti = 0.0; - for (auto algo: {PlanningAlgo::Order, PlanningAlgo::Estimate, PlanningAlgo::Cost, - PlanningAlgo::CostForceStrict}) { - auto res = benchmark_search(factory, num_docs + 1, true, false, false, 1.0, algo); - print_intermediate_blueprint_result(res, {a_hit_ratio, b_hit_ratio}, algo, num_docs); - if (algo == PlanningAlgo::Estimate) { - time_ms_esti = res.time_ms; - } - if (algo == PlanningAlgo::CostForceStrict) { - double speedup = time_ms_esti / res.time_ms; - if (speedup > max_speedup) { - max_speedup = speedup; + for (InFlow in_flow: in_flows) { + for (auto algo: {PlanningAlgo::Order, PlanningAlgo::Estimate, PlanningAlgo::Cost, + PlanningAlgo::CostForceStrict}) { + auto res = benchmark_search(*factory, num_docs + 1, in_flow.strict(), false, false, in_flow.rate(), algo); + print_intermediate_blueprint_result(res, {a_hit_ratio, b_hit_ratio}, algo, in_flow, num_docs); + if (algo == PlanningAlgo::Estimate) { + time_ms_esti = res.time_ms; } - if (speedup < min_speedup) { - min_speedup = speedup; + if (algo == PlanningAlgo::CostForceStrict) { + double speedup = time_ms_esti / res.time_ms; + if (speedup > max_speedup) { + max_speedup = speedup; + } + if (speedup < min_speedup) { + min_speedup = speedup; + } + std::cout << "speedup (esti/forc)=" << std::setprecision(4) << speedup << std::endl; } - std::cout << "speedup (esti/forc)=" << std::setprecision(4) << speedup << std::endl; } } } @@ -786,7 +802,19 @@ void run_and_benchmark(const BlueprintFactorySetup& a, const BlueprintFactorySetup& b, size_t num_docs) { std::cout << "AND[A={" << a.to_string() << "},B={" << b.to_string() << "}]" << std::endl; - run_intermediate_blueprint_benchmark<AndBlueprintFactory>(a, b, num_docs); + run_intermediate_blueprint_benchmark([](){ return std::make_unique<AndBlueprintFactory>(); }, {true}, a, b, num_docs); +} + +void +run_source_blender_benchmark(const BlueprintFactorySetup& a, const BlueprintFactorySetup& b, size_t num_docs) +{ + std::cout << "SB[A={" << a.to_string() << "},B={" << b.to_string() << "}]" << std::endl; + auto factory_factory = [&](){ + auto factory = std::make_unique<SourceBlenderBlueprintFactory>(); + factory->init_selector([](uint32_t i){ return (i%10 == 0) ? 1 : 2; }, num_docs + 1); + return factory; + }; + run_intermediate_blueprint_benchmark(factory_factory, {true, 0.75, 0.5, 0.25, 0.1, 0.01, 0.001}, a, b, num_docs); } //------------------------------------------------------------------------------------- @@ -970,16 +998,40 @@ TEST(IteratorBenchmark, analyze_AND_bitvector_vs_IN) } } +TEST(IteratorBenchmark, analyze_strict_SOURCEBLENDER_memory_and_disk) +{ + for (double small_ratio: {0.001, 0.005, 0.01, 0.05}) { + run_source_blender_benchmark({str_fs, QueryOperator::Term, {small_ratio}}, + {str_index, QueryOperator::Term, {small_ratio * 10}}, + num_docs); + } +} + TEST(IteratorBenchmark, analyze_OR_non_strict_fs) { for (auto or_hit_ratio : {0.01, 0.1, 0.5}) { BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::Or}, {false}, {or_hit_ratio}, {2, 4, 6, 8, 10, 100, 1000}); + //setup.force_strict = true; setup.filter_hit_ratios = gen_ratios(or_hit_ratio, 10.0, 13); run_benchmarks(setup); } } +TEST(IteratorBenchmark, analyze_OR_non_strict_fs_child_est_adjust) +{ + for (auto or_hit_ratio : {0.01, 0.1, 0.5}) { + for (uint32_t children : {2, 4, 6, 8, 10, 100, 1000}) { + double child_est = or_hit_ratio / children; + BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::Or}, {false}, {or_hit_ratio}, + {children}); + //setup.force_strict = true; + setup.filter_hit_ratios = gen_ratios(child_est, 10.0, 13); + run_benchmarks(setup); + } + } +} + TEST(IteratorBenchmark, analyze_OR_non_strict_non_fs) { BenchmarkSetup setup(num_docs, {int32}, {QueryOperator::Or}, {false}, {0.1}, {2, 4, 6, 8, 10}); @@ -1008,6 +1060,22 @@ TEST(IteratorBenchmark, analyze_btree_vs_bitvector_iterators_strict) run_benchmarks(setup); } +TEST(IteratorBenchmark, btree_vs_array_nonstrict_crossover) { + for (double hit_ratio: { 0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, + 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, + 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, + 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99, 1.0}) + { + auto btree = make_blueprint_factory(int32_array_fs, QueryOperator::Term, num_docs, 0, hit_ratio, 1, false); + auto array = make_blueprint_factory( int32_array, QueryOperator::Term, num_docs, 0, hit_ratio, 1, false); + auto time_ms = [&](auto &bpf, double in_flow) { + return Sample(benchmark_search(bpf, num_docs + 1, false, false, false, in_flow, PlanningAlgo::Cost).time_ms); + }; + auto calculate_at = [&](double in_flow) { return std::make_pair(time_ms(*btree, in_flow), time_ms(*array, in_flow)); }; + fprintf(stderr, "btree/array crossover@%5.3f: %8.6f\n", hit_ratio, find_crossover("TIME", "btree", "array", calculate_at, 0.0001)); + } +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); int res = RUN_ALL_TESTS(); diff --git a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp index ae2c0cac76f..94ecd8fa539 100644 --- a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp +++ b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp @@ -9,7 +9,6 @@ #include <vespa/searchlib/queryeval/andnotsearch.h> #include <vespa/searchlib/queryeval/andsearch.h> #include <vespa/searchlib/queryeval/dot_product_search.h> -#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/queryeval/orsearch.h> #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/queryeval/wand/weak_and_search.h> @@ -27,9 +26,9 @@ namespace { struct Writer { FILE *file; - Writer(const std::string &file_name) { + explicit Writer(const std::string &file_name) { file = fopen(file_name.c_str(), "w"); - assert(file != 0); + assert(file != nullptr); } void write(const char *data, size_t size) const { fwrite(data, 1, size, file); @@ -53,7 +52,7 @@ private: Writer _html; public: - Report(const std::string &file) : _html(file) { + explicit Report(const std::string &file) : _html(file) { _html.fmt("<html>\n"); _html.fmt("<head><title>Sparse Vector Search Benchmark Report</title></head>\n"); _html.fmt("<body>\n"); @@ -82,7 +81,7 @@ private: public: using UP = std::unique_ptr<Graph>; - Graph(const std::string &file) : _writer(file) {} + explicit Graph(const std::string &file) : _writer(file) {} void addValue(double x, double y) { _writer.fmt("%g %g\n", x, y); } }; @@ -98,8 +97,10 @@ private: public: using UP = std::unique_ptr<Plot>; - Plot(const std::string &title) : _name(vespalib::make_string("plot.%d", _plots++)), _graphs(0), - _writer(vespalib::make_string("%s.gnuplot", _name.c_str())) { + explicit Plot(const std::string &title) + : _name(vespalib::make_string("plot.%d", _plots++)), _graphs(0), + _writer(vespalib::make_string("%s.gnuplot", _name.c_str())) + { std::string png_file = vespalib::make_string("%s.png", _name.c_str()); _writer.fmt("set term png size 1200,800\n"); _writer.fmt("set output '%s'\n", png_file.c_str()); @@ -118,10 +119,10 @@ public: _writer.fmt("%s '%s' using 1:2 title '%s' w lines", (_graphs == 0) ? "plot " : ",", file.c_str(), legend.c_str()); ++_graphs; - return Graph::UP(new Graph(file)); + return std::make_unique<Graph>(file); } - static UP createPlot(const std::string &title) { return UP(new Plot(title)); } + static UP createPlot(const std::string &title) { return std::make_unique<Plot>(title); } }; int Plot::_plots = 0; @@ -137,19 +138,19 @@ struct ChildFactory { ChildFactory() {} virtual std::string name() const = 0; virtual SearchIterator::UP createChild(uint32_t idx, uint32_t limit) const = 0; - virtual ~ChildFactory() {} + virtual ~ChildFactory() = default; }; struct SparseVectorFactory { virtual std::string name() const = 0; virtual SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const = 0; - virtual ~SparseVectorFactory() {} + virtual ~SparseVectorFactory() = default; }; struct FilterStrategy { virtual std::string name() const = 0; virtual SearchIterator::UP createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const = 0; - virtual ~FilterStrategy() {} + virtual ~FilterStrategy() = default; }; //----------------------------------------------------------------------------- @@ -158,7 +159,7 @@ struct ModSearch : SearchIterator { uint32_t step; uint32_t limit; ModSearch(uint32_t step_in, uint32_t limit_in) : step(step_in), limit(limit_in) { setDocId(step); } - virtual void doSeek(uint32_t docid) override { + void doSeek(uint32_t docid) override { assert(docid > getDocId()); uint32_t hit = (docid / step) * step; if (hit < docid) { @@ -171,14 +172,14 @@ struct ModSearch : SearchIterator { setAtEnd(); } } - virtual void doUnpack(uint32_t) override {} + void doUnpack(uint32_t) override {} }; struct ModSearchFactory : ChildFactory { uint32_t bias; ModSearchFactory() : bias(1) {} explicit ModSearchFactory(int b) : bias(b) {} - virtual std::string name() const override { + std::string name() const override { return vespalib::make_string("ModSearch(%u)", bias); } SearchIterator::UP createChild(uint32_t idx, uint32_t limit) const override { @@ -190,14 +191,14 @@ struct ModSearchFactory : ChildFactory { struct VespaWandFactory : SparseVectorFactory { uint32_t n; - VespaWandFactory(uint32_t n_in) : n(n_in) {} - virtual std::string name() const override { + explicit VespaWandFactory(uint32_t n_in) noexcept : n(n_in) {} + std::string name() const override { return vespalib::make_string("VespaWand(%u)", n); } SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { wand::Terms terms; for (size_t i = 0; i < childCnt; ++i) { - terms.push_back(wand::Term(childFactory.createChild(i, limit), default_weight, limit / (i + 1))); + terms.emplace_back(childFactory.createChild(i, limit), default_weight, limit / (i + 1)); } return WeakAndSearch::create(terms, n, true); } @@ -205,16 +206,16 @@ struct VespaWandFactory : SparseVectorFactory { struct RiseWandFactory : SparseVectorFactory { uint32_t n; - RiseWandFactory(uint32_t n_in) : n(n_in) {} - virtual std::string name() const override { + explicit RiseWandFactory(uint32_t n_in) : n(n_in) {} + std::string name() const override { return vespalib::make_string("RiseWand(%u)", n); } SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { wand::Terms terms; for (size_t i = 0; i < childCnt; ++i) { - terms.push_back(wand::Term(childFactory.createChild(i, limit), default_weight, limit / (i + 1))); + terms.emplace_back(childFactory.createChild(i, limit), default_weight, limit / (i + 1)); } - return SearchIterator::UP(new rise::TermFrequencyRiseWand(terms, n)); + return std::make_unique<rise::TermFrequencyRiseWand>(terms, n); } }; @@ -230,7 +231,7 @@ struct WeightedSetFactory : SparseVectorFactory { tfmd.tagAsNotNeeded(); } } - virtual std::string name() const override { + std::string name() const override { return vespalib::make_string("WeightedSet%s%s", (field_is_filter ? "-filter" : ""), (tfmd.isNotNeeded() ? "-unranked" : "")); } SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { @@ -257,7 +258,7 @@ struct DotProductFactory : SparseVectorFactory { tfmd.tagAsNotNeeded(); } } - virtual std::string name() const override { + std::string name() const override { return vespalib::make_string("DotProduct%s%s", (field_is_filter ? "-filter" : ""), (tfmd.isNotNeeded() ? "-unranked" : "")); } SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { @@ -280,7 +281,7 @@ struct DotProductFactory : SparseVectorFactory { }; struct OrFactory : SparseVectorFactory { - virtual std::string name() const override { + std::string name() const override { return vespalib::make_string("Or"); } SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { @@ -295,7 +296,7 @@ struct OrFactory : SparseVectorFactory { //----------------------------------------------------------------------------- struct NoFilterStrategy : FilterStrategy { - virtual std::string name() const override { + std::string name() const override { return vespalib::make_string("NoFilter"); } SearchIterator::UP createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { @@ -332,8 +333,8 @@ struct NegativeFilterAfterStrategy : FilterStrategy { struct Result { vespalib::duration time; uint32_t num_hits; - Result() : time(max_time), num_hits(0) {} - Result(vespalib::duration t, uint32_t n) : time(t), num_hits(n) {} + Result() noexcept : time(max_time), num_hits(0) {} + Result(vespalib::duration t, uint32_t n) noexcept : time(t), num_hits(n) {} void combine(const Result &r) { if (time == max_time) { *this = r; @@ -357,7 +358,7 @@ Result run_single_benchmark(FilterStrategy &filterStrategy, SparseVectorFactory ++num_hits; sb.unpack(sb.getDocId()); } - return Result(timer.elapsed(), num_hits); + return {timer.elapsed(), num_hits}; } //----------------------------------------------------------------------------- @@ -384,8 +385,7 @@ public: void benchmark(SparseVectorFactory &svf, const std::vector<uint32_t> &child_counts) { Graph::UP graph = _plot->createGraph(svf.name()); fprintf(stderr, " search operator: %s\n", svf.name().c_str()); - for (size_t i = 0; i < child_counts.size(); ++i) { - uint32_t childCnt = child_counts[i]; + for (unsigned int childCnt : child_counts) { Result result; for (int j = 0; j < 5; ++j) { result.combine(run_single_benchmark(_filterStrategy, svf, _childFactory, childCnt, _limit)); diff --git a/searchlib/src/tests/queryeval/weak_and/rise_wand.h b/searchlib/src/tests/queryeval/weak_and/rise_wand.h index d4e66ec1907..4c7be54a6f0 100644 --- a/searchlib/src/tests/queryeval/weak_and/rise_wand.h +++ b/searchlib/src/tests/queryeval/weak_and/rise_wand.h @@ -15,8 +15,12 @@ namespace rise { struct TermFreqScorer { - static int64_t calculateMaxScore(const wand::Term &term) { - return TermFrequencyScorer::calculateMaxScore(term); + [[no_unique_address]] TermFrequencyScorer _termFrequencyScorer; + TermFreqScorer() noexcept + : _termFrequencyScorer() + { } + int64_t calculateMaxScore(const wand::Term &term) const noexcept { + return _termFrequencyScorer.calculateMaxScore(term); } static int64_t calculateScore(const wand::Term &term, uint32_t docId) { term.search->unpack(docId); @@ -43,9 +47,13 @@ private: //const addr_t *const *_streamPayloads; public: - StreamComparator(const docid_t *streamDocIds); + explicit StreamComparator(const docid_t *streamDocIds) noexcept + : _streamDocIds(streamDocIds) + { } //const addr_t *const *streamPayloads); - inline bool operator()(const uint16_t a, const uint16_t b); + bool operator()(const uint16_t a, const uint16_t b) const noexcept { + return (_streamDocIds[a] < _streamDocIds[b]); + } }; // number of streams present in the query @@ -66,6 +74,7 @@ private: // comparator that compares two streams StreamComparator _streamComparator; + [[no_unique_address]] Scorer _scorer; //------------------------------------------------------------------------- // variables used for scoring and pruning @@ -86,7 +95,7 @@ private: * * @return whether a valid pivot index is found */ - bool _findPivotFeatureIdx(const score_t threshold, uint32_t &pivotIdx); + bool _findPivotFeatureIdx(score_t threshold, uint32_t &pivotIdx); /** * let the first numStreamsToMove streams in the stream @@ -94,7 +103,7 @@ private: * * @param numStreamsToMove the number of streams that should move */ - void _moveStreamsAndSort(const uint32_t numStreamsToMove); + void _moveStreamsAndSort(uint32_t numStreamsToMove); /** * let the first numStreamsToMove streams in the stream @@ -106,7 +115,7 @@ private: * @param desiredDocId desired doc id * */ - void _moveStreamsToDocAndSort(const uint32_t numStreamsToMove, const docid_t desiredDocId); + void _moveStreamsToDocAndSort(uint32_t numStreamsToMove, docid_t desiredDocId); /** * do sort and merge for WAND @@ -115,18 +124,18 @@ private: * be sorted and then merge sort with the rest * */ - void _sortMerge(const uint32_t numStreamsToSort); + void _sortMerge(uint32_t numStreamsToSort); public: RiseWand(const Terms &terms, uint32_t n); - ~RiseWand(); + ~RiseWand() override; void next(); void doSeek(uint32_t docid) override; void doUnpack(uint32_t docid) override; }; -using TermFrequencyRiseWand = RiseWand<TermFreqScorer, std::greater_equal<uint64_t> >; -using DotProductRiseWand = RiseWand<DotProductScorer, std::greater<uint64_t> >; +using TermFrequencyRiseWand = RiseWand<TermFreqScorer, std::greater_equal<> >; +using DotProductRiseWand = RiseWand<DotProductScorer, std::greater<> >; } // namespacve rise diff --git a/searchlib/src/tests/queryeval/weak_and/rise_wand.hpp b/searchlib/src/tests/queryeval/weak_and/rise_wand.hpp index 32e17014f98..c477be5cc62 100644 --- a/searchlib/src/tests/queryeval/weak_and/rise_wand.hpp +++ b/searchlib/src/tests/queryeval/weak_and/rise_wand.hpp @@ -19,6 +19,7 @@ RiseWand<Scorer, Cmp>::RiseWand(const Terms &terms, uint32_t n) _streamIndices(new uint16_t[terms.size()]), _streamIndicesAux(new uint16_t[terms.size()]), _streamComparator(_streamDocIds), + _scorer(), _n(n), _limit(1), _streamScores(new score_t[terms.size()]), @@ -26,7 +27,7 @@ RiseWand<Scorer, Cmp>::RiseWand(const Terms &terms, uint32_t n) _terms(terms) { for (size_t i = 0; i < terms.size(); ++i) { - _terms[i].maxScore = Scorer::calculateMaxScore(terms[i]); + _terms[i].maxScore = _scorer.calculateMaxScore(terms[i]); _streamScores[i] = _terms[i].maxScore; _streams.push_back(terms[i].search); } @@ -46,8 +47,8 @@ RiseWand<Scorer, Cmp>::RiseWand(const Terms &terms, uint32_t n) template <typename Scorer, typename Cmp> RiseWand<Scorer, Cmp>::~RiseWand() { - for (size_t i = 0; i < _streams.size(); ++i) { - delete _streams[i]; + for (auto * stream : _streams) { + delete stream; } delete [] _streamScores; delete [] _streamIndicesAux; @@ -137,8 +138,7 @@ RiseWand<Scorer, Cmp>::_moveStreamsAndSort(const uint32_t numStreamsToMove) template <typename Scorer, typename Cmp> void -RiseWand<Scorer, Cmp>::_moveStreamsToDocAndSort(const uint32_t numStreamsToMove, - const docid_t desiredDocId) +RiseWand<Scorer, Cmp>::_moveStreamsToDocAndSort(const uint32_t numStreamsToMove, const docid_t desiredDocId) { for (uint32_t i=0; i<numStreamsToMove; ++i) { _streams[_streamIndices[i]]->seek(desiredDocId); @@ -195,7 +195,7 @@ RiseWand<Scorer, Cmp>::doUnpack(uint32_t docid) { score_t score = 0; for (size_t i = 0; i <= _lastPivotIdx; ++i) { - score += Scorer::calculateScore(_terms[_streamIndices[i]], docid); + score += _scorer.calculateScore(_terms[_streamIndices[i]], docid); } if (_scores.size() < _n || _scores.front() < score) { _scores.push(score); @@ -208,28 +208,5 @@ RiseWand<Scorer, Cmp>::doUnpack(uint32_t docid) } } -/** - ************ BEGIN STREAM COMPARTOR ********************* - */ -template <typename Scorer, typename Cmp> -RiseWand<Scorer, Cmp>::StreamComparator::StreamComparator( - const docid_t *streamDocIds) - : _streamDocIds(streamDocIds) -{ -} - -template <typename Scorer, typename Cmp> -inline bool -RiseWand<Scorer, Cmp>::StreamComparator::operator()(const uint16_t a, - const uint16_t b) -{ - if (_streamDocIds[a] < _streamDocIds[b]) return true; - return false; -} - -/** - ************ END STREAM COMPARTOR ********************* - */ - } // namespace rise diff --git a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp index 5e056eb6c0e..55dc3868ed4 100644 --- a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp +++ b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp @@ -29,17 +29,17 @@ struct Stats { size_t unpackCnt; size_t skippedDocs; size_t skippedHits; - Stats() : hitCnt(0), seekCnt(0), unpackCnt(0), + Stats() noexcept : hitCnt(0), seekCnt(0), unpackCnt(0), skippedDocs(0), skippedHits(0) {} - void hit() { + void hit() noexcept { ++hitCnt; } - void seek(size_t docs, size_t hits) { + void seek(size_t docs, size_t hits) noexcept { ++seekCnt; skippedDocs += docs; skippedHits += hits; } - void unpack() { + void unpack() noexcept { ++unpackCnt; } void print() { @@ -77,7 +77,7 @@ struct ModSearch : SearchIterator { } } void doUnpack(uint32_t docid) override { - if (tfmd != NULL) { + if (tfmd != nullptr) { tfmd->reset(docid); search::fef::TermFieldMatchDataPosition pos; pos.setElementWeight(info.getMaxWeight()); @@ -96,16 +96,16 @@ ModSearch::~ModSearch() = default; struct WandFactory { virtual std::string name() const = 0; virtual SearchIterator::UP create(const wand::Terms &terms) = 0; - virtual ~WandFactory() {} + virtual ~WandFactory() = default; }; struct VespaWandFactory : WandFactory { uint32_t n; - VespaWandFactory(uint32_t n_in) : n(n_in) {} + explicit VespaWandFactory(uint32_t n_in) noexcept : n(n_in) {} ~VespaWandFactory() override; - virtual std::string name() const override { return make_string("VESPA WAND (n=%u)", n); } - virtual SearchIterator::UP create(const wand::Terms &terms) override { - return SearchIterator::UP(WeakAndSearch::create(terms, n, true)); + std::string name() const override { return make_string("VESPA WAND (n=%u)", n); } + SearchIterator::UP create(const wand::Terms &terms) override { + return WeakAndSearch::create(terms, n, true); } }; @@ -113,11 +113,11 @@ VespaWandFactory::~VespaWandFactory() = default; struct VespaArrayWandFactory : WandFactory { uint32_t n; - VespaArrayWandFactory(uint32_t n_in) : n(n_in) {} + explicit VespaArrayWandFactory(uint32_t n_in) noexcept : n(n_in) {} ~VespaArrayWandFactory() override; - virtual std::string name() const override { return make_string("VESPA ARRAY WAND (n=%u)", n); } - virtual SearchIterator::UP create(const wand::Terms &terms) override { - return SearchIterator::UP(WeakAndSearch::createArrayWand(terms, n, true)); + std::string name() const override { return make_string("VESPA ARRAY WAND (n=%u)", n); } + SearchIterator::UP create(const wand::Terms &terms) override { + return WeakAndSearch::createArrayWand(terms, wand::TermFrequencyScorer(), n, true); } }; @@ -125,11 +125,11 @@ VespaArrayWandFactory::~VespaArrayWandFactory() = default; struct VespaHeapWandFactory : WandFactory { uint32_t n; - VespaHeapWandFactory(uint32_t n_in) : n(n_in) {} + explicit VespaHeapWandFactory(uint32_t n_in) noexcept : n(n_in) {} ~VespaHeapWandFactory() override; - virtual std::string name() const override { return make_string("VESPA HEAP WAND (n=%u)", n); } - virtual SearchIterator::UP create(const wand::Terms &terms) override { - return SearchIterator::UP(WeakAndSearch::createHeapWand(terms, n, true)); + std::string name() const override { return make_string("VESPA HEAP WAND (n=%u)", n); } + SearchIterator::UP create(const wand::Terms &terms) override { + return WeakAndSearch::createHeapWand(terms, wand::TermFrequencyScorer(), n, true); } }; @@ -138,39 +138,39 @@ VespaHeapWandFactory::~VespaHeapWandFactory() = default; struct VespaParallelWandFactory : public WandFactory { SharedWeakAndPriorityQueue scores; TermFieldMatchData rootMatchData; - VespaParallelWandFactory(uint32_t n) : scores(n), rootMatchData() {} + explicit VespaParallelWandFactory(uint32_t n) noexcept : scores(n), rootMatchData() {} ~VespaParallelWandFactory() override; - virtual std::string name() const override { return make_string("VESPA PWAND (n=%u)", scores.getScoresToTrack()); } - virtual SearchIterator::UP create(const wand::Terms &terms) override { - return SearchIterator::UP(ParallelWeakAndSearch::create(terms, + std::string name() const override { return make_string("VESPA PWAND (n=%u)", scores.getScoresToTrack()); } + SearchIterator::UP create(const wand::Terms &terms) override { + return ParallelWeakAndSearch::create(terms, PWMatchParams(scores, 0, 1, 1), - PWRankParams(rootMatchData, MatchData::UP()), true)); + PWRankParams(rootMatchData, {}), true); } }; VespaParallelWandFactory::~VespaParallelWandFactory() = default; struct VespaParallelArrayWandFactory : public VespaParallelWandFactory { - VespaParallelArrayWandFactory(uint32_t n) : VespaParallelWandFactory(n) {} + VespaParallelArrayWandFactory(uint32_t n) noexcept : VespaParallelWandFactory(n) {} ~VespaParallelArrayWandFactory() override; - virtual std::string name() const override { return make_string("VESPA ARRAY PWAND (n=%u)", scores.getScoresToTrack()); } - virtual SearchIterator::UP create(const wand::Terms &terms) override { - return SearchIterator::UP(ParallelWeakAndSearch::createArrayWand(terms, + std::string name() const override { return make_string("VESPA ARRAY PWAND (n=%u)", scores.getScoresToTrack()); } + SearchIterator::UP create(const wand::Terms &terms) override { + return ParallelWeakAndSearch::createArrayWand(terms, PWMatchParams(scores, 0, 1, 1), - PWRankParams(rootMatchData, MatchData::UP()), true)); + PWRankParams(rootMatchData, {}), true); } }; VespaParallelArrayWandFactory::~VespaParallelArrayWandFactory() = default; struct VespaParallelHeapWandFactory : public VespaParallelWandFactory { - VespaParallelHeapWandFactory(uint32_t n) : VespaParallelWandFactory(n) {} + explicit VespaParallelHeapWandFactory(uint32_t n) noexcept : VespaParallelWandFactory(n) {} ~VespaParallelHeapWandFactory() override; - virtual std::string name() const override { return make_string("VESPA HEAP PWAND (n=%u)", scores.getScoresToTrack()); } - virtual SearchIterator::UP create(const wand::Terms &terms) override { - return SearchIterator::UP(ParallelWeakAndSearch::createHeapWand(terms, + std::string name() const override { return make_string("VESPA HEAP PWAND (n=%u)", scores.getScoresToTrack()); } + SearchIterator::UP create(const wand::Terms &terms) override { + return ParallelWeakAndSearch::createHeapWand(terms, PWMatchParams(scores, 0, 1, 1), - PWRankParams(rootMatchData, MatchData::UP()), true)); + PWRankParams(rootMatchData, {}), true); } }; @@ -178,11 +178,11 @@ VespaParallelHeapWandFactory::~VespaParallelHeapWandFactory() = default; struct TermFrequencyRiseWandFactory : WandFactory { uint32_t n; - TermFrequencyRiseWandFactory(uint32_t n_in) : n(n_in) {} + explicit TermFrequencyRiseWandFactory(uint32_t n_in) noexcept : n(n_in) {} ~TermFrequencyRiseWandFactory() override; - virtual std::string name() const override { return make_string("RISE WAND TF (n=%u)", n); } - virtual SearchIterator::UP create(const wand::Terms &terms) override { - return SearchIterator::UP(new rise::TermFrequencyRiseWand(terms, n)); + std::string name() const override { return make_string("RISE WAND TF (n=%u)", n); } + SearchIterator::UP create(const wand::Terms &terms) override { + return std::make_unique<rise::TermFrequencyRiseWand>(terms, n); } }; @@ -190,11 +190,11 @@ TermFrequencyRiseWandFactory::~TermFrequencyRiseWandFactory() = default; struct DotProductRiseWandFactory : WandFactory { uint32_t n; - DotProductRiseWandFactory(uint32_t n_in) : n(n_in) {} + explicit DotProductRiseWandFactory(uint32_t n_in) noexcept : n(n_in) {} ~DotProductRiseWandFactory() override; - virtual std::string name() const override { return make_string("RISE WAND DP (n=%u)", n); } - virtual SearchIterator::UP create(const wand::Terms &terms) override { - return SearchIterator::UP(new rise::DotProductRiseWand(terms, n)); + std::string name() const override { return make_string("RISE WAND DP (n=%u)", n); } + SearchIterator::UP create(const wand::Terms &terms) override { + return std::make_unique<rise::DotProductRiseWand>(terms, n); } }; @@ -204,13 +204,13 @@ struct FilterFactory : WandFactory { WandFactory &factory; Stats stats; uint32_t n; - FilterFactory(WandFactory &f, uint32_t n_in) : factory(f), n(n_in) {} + FilterFactory(WandFactory &f, uint32_t n_in) noexcept : factory(f), n(n_in) {} ~FilterFactory() override; - virtual std::string name() const override { return make_string("Filter (mod=%u) [%s]", n, factory.name().c_str()); } - virtual SearchIterator::UP create(const wand::Terms &terms) override { + std::string name() const override { return make_string("Filter (mod=%u) [%s]", n, factory.name().c_str()); } + SearchIterator::UP create(const wand::Terms &terms) override { AndNotSearch::Children children; children.push_back(factory.create(terms)); - children.emplace_back(new ModSearch(stats, n, search::endDocId, n, NULL)); + children.emplace_back(new ModSearch(stats, n, search::endDocId, n, nullptr)); return AndNotSearch::create(std::move(children), true); } }; @@ -220,8 +220,8 @@ FilterFactory::~FilterFactory() = default; struct Setup { Stats stats; vespalib::duration minTime; - Setup() : stats(), minTime(10000s) {} - virtual ~Setup() {} + Setup() noexcept : stats(), minTime(10000s) {} + virtual ~Setup() = default; virtual std::string name() const = 0; virtual SearchIterator::UP create() = 0; void perform() { @@ -256,10 +256,10 @@ struct WandSetup : Setup { MatchData::UP matchData; WandSetup(WandFactory &f, uint32_t c, uint32_t l) : Setup(), factory(f), childCnt(c), limit(l), weight(100), matchData() {} ~WandSetup() override; - virtual std::string name() const override { + std::string name() const override { return make_string("Wand Setup (terms=%u,docs=%u) [%s]", childCnt, limit, factory.name().c_str()); } - virtual SearchIterator::UP create() override { + SearchIterator::UP create() override { MatchDataLayout layout; std::vector<TermFieldHandle> handles; for (size_t i = 0; i < childCnt; ++i) { diff --git a/searchlib/src/tests/queryeval/weak_and_scorers/weak_and_scorers_test.cpp b/searchlib/src/tests/queryeval/weak_and_scorers/weak_and_scorers_test.cpp index 528e117f976..8a0bc28f4dd 100644 --- a/searchlib/src/tests/queryeval/weak_and_scorers/weak_and_scorers_test.cpp +++ b/searchlib/src/tests/queryeval/weak_and_scorers/weak_and_scorers_test.cpp @@ -25,18 +25,18 @@ struct TestIterator : public SearchIterator _useInfo(useInfo), _unpackDocId(0) {} - virtual void doSeek(uint32_t docId) override { + void doSeek(uint32_t docId) override { (void) docId; } - virtual void doUnpack(uint32_t docId) override { + void doUnpack(uint32_t docId) override { _unpackDocId = docId; _tfmd.appendPosition(TermFieldMatchDataPosition(0, 0, _termWeight, 1)); } - virtual const PostingInfo *getPostingInfo() const override { - return (_useInfo ? &_info : NULL); + const PostingInfo *getPostingInfo() const override { + return (_useInfo ? &_info : nullptr); } static UP create(int32_t maxWeight, int32_t termWeight, bool useInfo) { - return UP(new TestIterator(maxWeight, termWeight, useInfo)); + return std::make_unique<TestIterator>(maxWeight, termWeight, useInfo); } }; @@ -63,4 +63,27 @@ TEST("require that DotProductScorer calculates term score") EXPECT_EQUAL(11u, itr->_unpackDocId); } +TEST("test bm25 idf scorer for wand") +{ + wand::Bm25TermFrequencyScorer scorer(1000000, 1.0); + EXPECT_EQUAL(13410046, scorer.calculateMaxScore(1, 1)); + EXPECT_EQUAL(11464136, scorer.calculateMaxScore(10, 1)); + EXPECT_EQUAL(6907256, scorer.calculateMaxScore(1000, 1)); + EXPECT_EQUAL(4605121, scorer.calculateMaxScore(10000, 1)); + EXPECT_EQUAL(2302581, scorer.calculateMaxScore(100000, 1)); + EXPECT_EQUAL(693147, scorer.calculateMaxScore(500000, 1)); + EXPECT_EQUAL(105360, scorer.calculateMaxScore(900000, 1)); + EXPECT_EQUAL(10050, scorer.calculateMaxScore(990000, 1)); +} + +TEST("test limited range of bm25 idf scorer for wand") +{ + wand::Bm25TermFrequencyScorer scorer08(1000000, 0.8); + wand::Bm25TermFrequencyScorer scorer10(1000000, 1.0); + EXPECT_EQUAL(8207814, scorer08.calculateMaxScore(1000, 1)); + EXPECT_EQUAL(2690049, scorer08.calculateMaxScore(990000, 1)); + EXPECT_EQUAL(6907256, scorer10.calculateMaxScore(1000, 1)); + EXPECT_EQUAL(10050, scorer10.calculateMaxScore(990000, 1)); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index d50677314df..a1cf86c95cc 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -936,6 +936,36 @@ TYPED_TEST(HnswIndexTest, search_during_remove) this->expect_top_3_by_docid("{0, 0}", {0, 0}, {7}); } +TYPED_TEST(HnswIndexTest, inconsistent_index) +{ + this->init(false); + this->vectors.clear(); + this->vectors.set(1, {1, 3}).set(2, {7, 1}).set(3, {6, 5}).set(4, {8, 3}).set(5, {10, 3}); + this->add_document(1); + this->add_document(2); + this->add_document(3); + this->add_document(4); + this->add_document(5); + this->expect_entry_point(1, 0); + this->expect_level_0(1, {2, 3}); + this->expect_level_0(2, {1, 3, 4, 5}); + this->expect_level_0(3, {1, 2, 4}); + this->expect_level_0(4, {2, 3, 5}); + this->expect_level_0(5, {2, 4}); + EXPECT_EQ(0, this->index->check_consistency(6)); + // Remove vector for docid 5 but don't update index. + this->vectors.clear(5); + EXPECT_EQ(1, this->index->check_consistency(6)); + /* + * Removing document 2 causes mutual reconnect for nodes [1, 3, 4, 5] + * where nodes 1 and 5 are not previously connected. Distance from + * node 1 to node 5 cannot be calculated due to missing vector. + */ + this->remove_document(2); + // No reconnect for node without vector + this->expect_level_0(5, {4}); +} + using HnswMultiIndexTest = HnswIndexTest<HnswIndex<HnswIndexType::MULTI>>; namespace { diff --git a/searchlib/src/tests/util/token_extractor/token_extractor_test.cpp b/searchlib/src/tests/util/token_extractor/token_extractor_test.cpp index e6944e257e9..5eb42bb8ac4 100644 --- a/searchlib/src/tests/util/token_extractor/token_extractor_test.cpp +++ b/searchlib/src/tests/util/token_extractor/token_extractor_test.cpp @@ -118,7 +118,7 @@ TEST_F(TokenExtractorTest, empty_string) TEST_F(TokenExtractorTest, plain_string) { - EXPECT_EQ((Words{"Plain string"}), process(StringFieldValue("Plain string"))); + EXPECT_EQ((Words{}), process(StringFieldValue("Plain string"))); } TEST_F(TokenExtractorTest, normal_string) diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 5b17b491a20..70b86bf22a1 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -94,6 +94,7 @@ using search::queryeval::StrictHeapOrSearch; using search::queryeval::WeightedSetTermBlueprint; using search::queryeval::flow::btree_cost; using search::queryeval::flow::btree_strict_cost; +using search::queryeval::flow::estimate_when_unknown; using search::queryeval::flow::get_num_indirections; using search::queryeval::flow::lookup_cost; using search::queryeval::flow::lookup_strict_cost; @@ -150,10 +151,9 @@ public: search::queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override { if (_hit_estimate.is_unknown()) { // E.g. attributes without fast-search are not able to provide a hit estimate. - // In this case we just assume matching half of the document corpus. // In addition, matching is lookup based, and we are not able to skip documents efficiently when being strict. size_t indirections = get_num_indirections(_attr.getBasicType(), _attr.getCollectionType()); - return {0.5, lookup_cost(indirections), lookup_strict_cost(indirections)}; + return {estimate_when_unknown(), lookup_cost(indirections), lookup_strict_cost(indirections)}; } else { double rel_est = abs_to_rel_est(_hit_estimate.est_hits(), docid_limit); return {rel_est, btree_cost(rel_est), btree_strict_cost(rel_est)}; diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_params.h b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_params.h index e2928710a32..ac6fc6f603a 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_params.h +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_params.h @@ -16,15 +16,18 @@ struct AttributeBlueprintParams double global_filter_upper_limit; double target_hits_max_adjustment_factor; vespalib::FuzzyMatchingAlgorithm fuzzy_matching_algorithm; + double weakand_range; AttributeBlueprintParams(double global_filter_lower_limit_in, double global_filter_upper_limit_in, double target_hits_max_adjustment_factor_in, - vespalib::FuzzyMatchingAlgorithm fuzzy_matching_algorithm_in) + vespalib::FuzzyMatchingAlgorithm fuzzy_matching_algorithm_in, + double weakand_range_in) : global_filter_lower_limit(global_filter_lower_limit_in), global_filter_upper_limit(global_filter_upper_limit_in), target_hits_max_adjustment_factor(target_hits_max_adjustment_factor_in), - fuzzy_matching_algorithm(fuzzy_matching_algorithm_in) + fuzzy_matching_algorithm(fuzzy_matching_algorithm_in), + weakand_range(weakand_range_in) { } @@ -32,7 +35,8 @@ struct AttributeBlueprintParams : AttributeBlueprintParams(fef::indexproperties::matching::GlobalFilterLowerLimit::DEFAULT_VALUE, fef::indexproperties::matching::GlobalFilterUpperLimit::DEFAULT_VALUE, fef::indexproperties::matching::TargetHitsMaxAdjustmentFactor::DEFAULT_VALUE, - fef::indexproperties::matching::FuzzyAlgorithm::DEFAULT_VALUE) + fef::indexproperties::matching::FuzzyAlgorithm::DEFAULT_VALUE, + fef::indexproperties::temporary::WeakAndRange::DEFAULT_VALUE) { } }; diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp index 505b8166ee7..03d2e94b5d0 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp @@ -68,7 +68,7 @@ Bm25Executor::Bm25Executor(const fef::FieldInfo& field, } double -Bm25Executor::calculate_inverse_document_frequency(uint32_t matching_doc_count, uint32_t total_doc_count) +Bm25Executor::calculate_inverse_document_frequency(uint32_t matching_doc_count, uint32_t total_doc_count) noexcept { return std::log(1 + (static_cast<double>(total_doc_count - matching_doc_count + 0.5) / static_cast<double>(matching_doc_count + 0.5))); diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.h b/searchlib/src/vespa/searchlib/features/bm25_feature.h index a1b45375285..637d656990b 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.h +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.h @@ -39,7 +39,7 @@ public: double k1_param, double b_param); - double static calculate_inverse_document_frequency(uint32_t matching_doc_count, uint32_t total_doc_count); + double static calculate_inverse_document_frequency(uint32_t matching_doc_count, uint32_t total_doc_count) noexcept; void handle_bind_match_data(const fef::MatchData& match_data) override; void execute(uint32_t docId) override; diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp index 4637ad5a4e8..1f88c34bef3 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp @@ -179,6 +179,21 @@ namespace onsummary { namespace temporary { +const vespalib::string WeakAndRange::NAME("vespa.weakand.range"); +const double WeakAndRange::DEFAULT_VALUE(0.0); + +double +WeakAndRange::lookup(const Properties &props) +{ + return lookup(props, DEFAULT_VALUE); +} + +double +WeakAndRange::lookup(const Properties &props, double defaultValue) +{ + return lookupDouble(props, NAME, defaultValue); +} + } namespace mutate { diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h index db8de8209a9..d047eb13347 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.h +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h @@ -178,6 +178,18 @@ namespace mutate { // Add temporary flags used for safe rollout of new features here namespace temporary { +/** + * A number in the range [0,1] for the effective idf range for WeakAndOperator. + * 1.0 will give the complete range as used by default by bm25. + * scaled_idf = (1.0 - range) * max_idf + (range * idf) + * 0.0 which is default gives default legacy behavior. + **/ +struct WeakAndRange { + static const vespalib::string NAME; + static const double DEFAULT_VALUE; + static double lookup(const Properties &props); + static double lookup(const Properties &props, double defaultValue); +}; } namespace mutate::on_match { diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index aadc5300ede..25588cf3229 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -71,6 +71,7 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _global_filter_lower_limit(0.0), _global_filter_upper_limit(1.0), _target_hits_max_adjustment_factor(20.0), + _weakand_range(0.0), _fuzzy_matching_algorithm(vespalib::FuzzyMatchingAlgorithm::DfaTable), _mutateOnMatch(), _mutateOnFirstPhase(), @@ -126,6 +127,7 @@ RankSetup::configure() set_global_filter_upper_limit(matching::GlobalFilterUpperLimit::lookup(_indexEnv.getProperties())); set_target_hits_max_adjustment_factor(matching::TargetHitsMaxAdjustmentFactor::lookup(_indexEnv.getProperties())); set_fuzzy_matching_algorithm(matching::FuzzyAlgorithm::lookup(_indexEnv.getProperties())); + set_weakand_range(temporary::WeakAndRange::lookup(_indexEnv.getProperties())); _mutateOnMatch._attribute = mutate::on_match::Attribute::lookup(_indexEnv.getProperties()); _mutateOnMatch._operation = mutate::on_match::Operation::lookup(_indexEnv.getProperties()); _mutateOnFirstPhase._attribute = mutate::on_first_phase::Attribute::lookup(_indexEnv.getProperties()); diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index d8b977a0331..f20ecd4b42b 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -80,6 +80,7 @@ private: double _global_filter_lower_limit; double _global_filter_upper_limit; double _target_hits_max_adjustment_factor; + double _weakand_range; vespalib::FuzzyMatchingAlgorithm _fuzzy_matching_algorithm; MutateOperation _mutateOnMatch; MutateOperation _mutateOnFirstPhase; @@ -402,6 +403,8 @@ public: double get_target_hits_max_adjustment_factor() const { return _target_hits_max_adjustment_factor; } void set_fuzzy_matching_algorithm(vespalib::FuzzyMatchingAlgorithm v) { _fuzzy_matching_algorithm = v; } vespalib::FuzzyMatchingAlgorithm get_fuzzy_matching_algorithm() const { return _fuzzy_matching_algorithm; } + void set_weakand_range(double v) { _weakand_range = v; } + double get_weakand_range() const { return _weakand_range; } /** * This method may be used to indicate that certain features diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 7334db4b716..cfa165be067 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -1,14 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "blueprint.h" -#include "leaf_blueprints.h" +#include "andnotsearch.h" +#include "andsearch.h" #include "emptysearch.h" -#include "full_search.h" #include "field_spec.hpp" -#include "andsearch.h" -#include "orsearch.h" -#include "andnotsearch.h" +#include "flow_tuning.h" +#include "full_search.h" +#include "leaf_blueprints.h" #include "matching_elements_search.h" +#include "orsearch.h" #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/objects/visit.hpp> #include <vespa/vespalib/objects/objectdumper.h> @@ -238,7 +239,7 @@ Blueprint::default_flow_stats(uint32_t docid_limit, uint32_t abs_est, size_t chi FlowStats Blueprint::default_flow_stats(size_t child_cnt) { - return {0.5, 1.0 + child_cnt, 1.0 + child_cnt}; + return {flow::estimate_when_unknown(), 1.0 + child_cnt, 1.0 + child_cnt}; } std::unique_ptr<MatchingElementsSearch> diff --git a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h index cf1d1a8c09f..5ed61ef9fc8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h +++ b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h @@ -60,6 +60,12 @@ inline size_t get_num_indirections(const attribute::BasicType& basic_type, return res; } +// Some blueprints are not able to provide a hit estimate (e.g. attributes without fast-search). +// In such cases the following estimate is used instead. In most cases this is an overestimate. +inline double estimate_when_unknown() { + return 0.1; +} + // Non-strict cost of lookup based matching in an attribute (not fast-search). // Test used: IteratorBenchmark::analyze_term_search_in_attributes_non_strict inline double lookup_cost(size_t num_indirections) { @@ -90,7 +96,7 @@ inline double lookup_strict_cost(size_t num_indirections) { * as the latency (time) penalty is higher if choosing wrong. */ inline double non_strict_cost_of_strict_iterator(double estimate, double strict_cost) { - return strict_cost + strict_cost_diff(estimate, 1.0); + return 2.0 * (strict_cost + strict_cost_diff(estimate, 0.5)); } // Strict cost of matching in a btree posting list (e.g. fast-search attribute or memory index field). diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index 33b249572f0..b0b3b302e82 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -492,7 +492,9 @@ WeakAndBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, _weights[i], getChild(i).getState().estimate().estHits); } - return WeakAndSearch::create(terms, _n, strict()); + return (_idf_range == 0.0) + ? WeakAndSearch::create(terms, wand::TermFrequencyScorer(), _n, strict()) + : WeakAndSearch::create(terms, wand::Bm25TermFrequencyScorer(get_docid_limit(), _idf_range), _n, strict()); } SearchIterator::UP @@ -756,7 +758,18 @@ SourceBlenderBlueprint::calculate_flow_stats(uint32_t) const { my_cost = std::max(my_cost, child->cost()); my_strict_cost = std::max(my_strict_cost, child->strict_cost()); } - return {OrFlow::estimate_of(get_children()), my_cost, my_strict_cost}; + double my_est = OrFlow::estimate_of(get_children()); + return {my_est, my_cost + 1.0, my_strict_cost + my_est}; +} + +double +SourceBlenderBlueprint::estimate_self_cost(InFlow in_flow) const noexcept +{ + if (in_flow.strict()) { + return estimate(); + } else { + return in_flow.rate(); + } } Blueprint::HitEstimate diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h index ade4c9318e4..f7eeace3e8b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h @@ -90,6 +90,7 @@ class WeakAndBlueprint : public IntermediateBlueprint { private: uint32_t _n; + float _idf_range; std::vector<uint32_t> _weights; AnyFlow my_flow(InFlow in_flow) const override; @@ -107,7 +108,8 @@ public: fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(FilterConstraint constraint) const override; - explicit WeakAndBlueprint(uint32_t n) noexcept : _n(n) {} + explicit WeakAndBlueprint(uint32_t n) noexcept : WeakAndBlueprint(n, 0.0) {} + WeakAndBlueprint(uint32_t n, float idf_range) noexcept : _n(n), _idf_range(idf_range), _weights() {} ~WeakAndBlueprint() override; void addTerm(Blueprint::UP bp, uint32_t weight) { addChild(std::move(bp)); @@ -199,6 +201,7 @@ public: explicit SourceBlenderBlueprint(const ISourceSelector &selector) noexcept; ~SourceBlenderBlueprint() override; FlowStats calculate_flow_stats(uint32_t docid_limit) const final; + double estimate_self_cost(InFlow in_flow) const noexcept override; HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; void sort(Children &children, InFlow in_flow) const override; diff --git a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp index d825c9e1a20..9d19ba87af7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp @@ -11,9 +11,9 @@ namespace search::queryeval { //----------------------------------------------------------------------------- FlowStats -EmptyBlueprint::calculate_flow_stats(uint32_t docid_limit) const +EmptyBlueprint::calculate_flow_stats(uint32_t) const { - return default_flow_stats(docid_limit, 0, 0); + return {0.0, 0.2, 0.0}; } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h index ed8d4b4e4ac..88f0c9288f9 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h @@ -2,10 +2,9 @@ #pragma once -#include <algorithm> -#include <cmath> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/searchlib/features/bm25_feature.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/queryeval/iterator_pack.h> #include <vespa/searchlib/attribute/posting_iterator_pack.h> @@ -13,20 +12,16 @@ #include <vespa/vespalib/util/priority_queue.h> #include <vespa/searchlib/attribute/i_docid_with_weight_posting_store.h> #include <vespa/vespalib/util/stringfmt.h> +#include <cmath> namespace search::queryeval::wand { //----------------------------------------------------------------------------- -struct Term; -using Terms = std::vector<Term>; using score_t = int64_t; using docid_t = uint32_t; using ref_t = uint16_t; -using Attr = IDirectPostingStore; -using AttrDictEntry = Attr::LookupResult; - //----------------------------------------------------------------------------- /** @@ -46,7 +41,7 @@ struct Term { Term(SearchIterator *s, int32_t w, uint32_t e) noexcept : Term(s, w, e, nullptr) {} Term(SearchIterator::UP s, int32_t w, uint32_t e) noexcept : Term(s.release(), w, e, nullptr) {} }; - +using Terms = std::vector<Term>; //----------------------------------------------------------------------------- // input manipulation utilities @@ -75,7 +70,7 @@ auto assemble(const F &f, const Order &order)->std::vector<decltype(f(0))> { } int32_t get_max_weight(const SearchIterator &search) { - const MinMaxPostingInfo *minMax = dynamic_cast<const MinMaxPostingInfo *>(search.getPostingInfo()); + const auto *minMax = dynamic_cast<const MinMaxPostingInfo *>(search.getPostingInfo()); return (minMax != nullptr) ? minMax->getMaxWeight() : std::numeric_limits<int32_t>::max(); } @@ -163,7 +158,7 @@ public: ~VectorizedState(); template <typename Scorer, typename Input> - std::vector<ref_t> init_state(const Input &input, uint32_t docIdLimit); + std::vector<ref_t> init_state(const Input &input, const Scorer & scorer, uint32_t docIdLimit); docid_t *docId() { return &(_docId[0]); } const int32_t *weight() const { return &(_weight[0]); } @@ -202,14 +197,14 @@ VectorizedState<IteratorPack>::operator=(VectorizedState &&) noexcept = default; template <typename IteratorPack> template <typename Scorer, typename Input> std::vector<ref_t> -VectorizedState<IteratorPack>::init_state(const Input &input, uint32_t docIdLimit) { +VectorizedState<IteratorPack>::init_state(const Input &input, const Scorer & scorer, uint32_t docIdLimit) { std::vector<ref_t> order; std::vector<score_t> max_scores; order.reserve(input.size()); max_scores.reserve(input.size()); for (size_t i = 0; i < input.size(); ++i) { order.push_back(i); - max_scores.push_back(Scorer::calculate_max_score(input, i)); + max_scores.push_back(scorer.calculate_max_score(input, i)); } std::sort(order.begin(), order.end(), MaxSkipOrder<Input>(docIdLimit, input, max_scores)); _docId = assemble([&input](ref_t ref){ return input.get_initial_docid(ref); }, order); @@ -238,7 +233,7 @@ private: public: template <typename Scorer> - VectorizedIteratorTerms(const Terms &t, const Scorer &, uint32_t docIdLimit, + VectorizedIteratorTerms(const Terms &t, const Scorer & scorer, uint32_t docIdLimit, fef::MatchData::UP childrenMatchData); VectorizedIteratorTerms(VectorizedIteratorTerms &&) noexcept; VectorizedIteratorTerms & operator=(VectorizedIteratorTerms &&) noexcept; @@ -250,11 +245,11 @@ public: }; template <typename Scorer> -VectorizedIteratorTerms::VectorizedIteratorTerms(const Terms &t, const Scorer &, uint32_t docIdLimit, +VectorizedIteratorTerms::VectorizedIteratorTerms(const Terms &t, const Scorer & scorer, uint32_t docIdLimit, fef::MatchData::UP childrenMatchData) : _terms() { - std::vector<ref_t> order = init_state<Scorer>(TermInput(t), docIdLimit); + std::vector<ref_t> order = init_state<Scorer>(TermInput(t), scorer, docIdLimit); _terms = assemble([&t](ref_t ref){ return t[ref]; }, order); iteratorPack() = SearchIteratorPack(assemble([&t](ref_t ref){ return t[ref].search; }, order), assemble([&t](ref_t ref){ return t[ref].matchData; }, order), @@ -268,10 +263,10 @@ struct VectorizedAttributeTerms : VectorizedState<DocidWithWeightIteratorPack> { VectorizedAttributeTerms(const std::vector<int32_t> &weights, const std::vector<IDirectPostingStore::LookupResult> &dict_entries, const IDocidWithWeightPostingStore &attr, - const Scorer &, + const Scorer & scorer, docid_t docIdLimit) { - std::vector<ref_t> order = init_state<Scorer>(AttrInput(weights, dict_entries), docIdLimit); + std::vector<ref_t> order = init_state<Scorer>(AttrInput(weights, dict_entries), scorer, docIdLimit); std::vector<DocidWithWeightIterator> iterators; iterators.reserve(order.size()); for (size_t i = 0; i < order.size(); ++i) { @@ -291,7 +286,7 @@ struct VectorizedAttributeTerms : VectorizedState<DocidWithWeightIteratorPack> { **/ struct DocIdOrder { const docid_t *termPos; - explicit DocIdOrder(docid_t *pos) noexcept : termPos(pos) {} + explicit DocIdOrder(const docid_t *pos) noexcept : termPos(pos) {} bool at_end(ref_t ref) const noexcept { return termPos[ref] == search::endDocId; } docid_t get_pos(ref_t ref) const noexcept { return termPos[ref]; } bool operator()(ref_t a, ref_t b) const noexcept { @@ -389,7 +384,7 @@ DualHeap<FutureHeap, PastHeap>::stringify() const { } //----------------------------------------------------------------------------- -#define TermFrequencyScorer_TERM_SCORE_FACTOR 1000000.0 +constexpr double TermFrequencyScorer_TERM_SCORE_FACTOR = 1000000.0; /** * Scorer used with WeakAndAlgorithm that calculates a pseudo term frequency @@ -398,18 +393,50 @@ DualHeap<FutureHeap, PastHeap>::stringify() const { struct TermFrequencyScorer { // weight * idf, scaled to fixedpoint - static score_t calculateMaxScore(double estHits, double weight) noexcept { + score_t calculateMaxScore(double estHits, double weight) const noexcept { return (score_t) (TermFrequencyScorer_TERM_SCORE_FACTOR * weight / (1.0 + log(1.0 + (estHits / 1000.0)))); } - static score_t calculateMaxScore(const Term &term) noexcept { + score_t calculateMaxScore(const Term &term) const noexcept { return calculateMaxScore(term.estHits, term.weight) + 1; } template <typename Input> - static score_t calculate_max_score(const Input &input, ref_t ref) { + score_t calculate_max_score(const Input &input, ref_t ref) const noexcept { + return calculateMaxScore(input.get_est_hits(ref), input.get_weight(ref)) + 1; + } +}; + +class Bm25TermFrequencyScorer +{ +public: + using Bm25Executor = features::Bm25Executor; + Bm25TermFrequencyScorer(uint32_t num_docs, float range) noexcept + : _num_docs(num_docs), + _range(range), + _max_idf(Bm25Executor::calculate_inverse_document_frequency(1, _num_docs)) + { } + double apply_range(double idf) const noexcept { + return (1.0 - _range)*_max_idf + _range * idf; + } + // weight * scaled_bm25_idf, scaled to fixedpoint + score_t calculateMaxScore(double estHits, double weight) const noexcept { + return score_t(TermFrequencyScorer_TERM_SCORE_FACTOR * weight * + apply_range(Bm25Executor::calculate_inverse_document_frequency(estHits, _num_docs))); + } + + score_t calculateMaxScore(const Term &term) const noexcept { + return calculateMaxScore(term.estHits, term.weight) + 1; + } + + template <typename Input> + score_t calculate_max_score(const Input &input, ref_t ref) const noexcept { return calculateMaxScore(input.get_est_hits(ref), input.get_weight(ref)) + 1; } +private: + uint32_t _num_docs; + float _range; + double _max_idf; }; //----------------------------------------------------------------------------- @@ -453,14 +480,14 @@ struct DotProductScorer // used with parallel wand where we can safely discard hits based on score struct GreaterThan { score_t threshold; - GreaterThan(score_t t) : threshold(t) {} + explicit GreaterThan(score_t t) noexcept : threshold(t) {} bool operator()(score_t score) const { return (score > threshold); } }; // used with old-style vespa wand to ensure at least AND'ish results struct GreaterThanEqual { score_t threshold; - GreaterThanEqual(score_t t) : threshold(t) {} + explicit GreaterThanEqual(score_t t) noexcept : threshold(t) {} bool operator()(score_t score) const { return (score >= threshold); } }; @@ -521,10 +548,10 @@ private: } template <typename VectorizedTerms, typename Heaps, typename Scorer, typename AboveThreshold> - bool check_present_score(VectorizedTerms &terms, Heaps &heaps, score_t &max_score, const Scorer &, AboveThreshold &&aboveThreshold) { + bool check_present_score(VectorizedTerms &terms, Heaps &heaps, score_t &max_score, const Scorer & scorer, AboveThreshold &&aboveThreshold) { ref_t *end = heaps.present_end(); for (ref_t *ref = heaps.present_begin(); ref != end; ++ref) { - score_t term_score = Scorer::calculateScore(terms, *ref, _candidate); + score_t term_score = scorer.calculateScore(terms, *ref, _candidate); _partial_score += term_score; max_score -= (terms.maxScore(*ref) - term_score); if (!aboveThreshold(max_score)) { @@ -535,11 +562,11 @@ private: } template <typename VectorizedTerms, typename Heaps, typename Scorer, typename AboveThreshold> - bool check_past_score(VectorizedTerms &terms, Heaps &heaps, score_t &max_score, const Scorer &, AboveThreshold &&aboveThreshold) { + bool check_past_score(VectorizedTerms &terms, Heaps &heaps, score_t &max_score, const Scorer & scorer, AboveThreshold &&aboveThreshold) { while (heaps.has_past() && !aboveThreshold(_partial_score)) { heaps.pop_past(); if (step_term(terms, heaps.last_present())) { - score_t term_score = Scorer::calculateScore(terms, heaps.last_present(), _candidate); + score_t term_score = scorer.calculateScore(terms, heaps.last_present(), _candidate); _partial_score += term_score; max_score -= (terms.maxScore(heaps.last_present()) - term_score); } else { @@ -618,7 +645,7 @@ public: } template <typename VectorizedTerms, typename Heaps, typename Scorer, typename AboveThreshold> - bool check_score(VectorizedTerms &terms, Heaps &heaps, Scorer &&scorer, AboveThreshold &&aboveThreshold) { + bool check_score(VectorizedTerms &terms, Heaps &heaps, const Scorer &scorer, AboveThreshold &&aboveThreshold) { _partial_score = 0; score_t max_score = _maxUpperBound; if (check_present_score(terms, heaps, max_score, scorer, aboveThreshold)) { @@ -630,12 +657,12 @@ public: } template <typename VectorizedTerms, typename Heaps, typename Scorer> - score_t get_full_score(VectorizedTerms &terms, Heaps &heaps, Scorer &&) { + score_t get_full_score(VectorizedTerms &terms, Heaps &heaps, const Scorer & scorer) { score_t score = _partial_score; while (heaps.has_past()) { heaps.pop_any_past(); if (step_term(terms, heaps.last_present())) { - score += Scorer::calculateScore(terms, heaps.last_present(), _candidate); + score += scorer.calculateScore(terms, heaps.last_present(), _candidate); } else { evict_last_present(terms, heaps); } diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp index 375a6598b49..cf3fd44ad4f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp @@ -42,11 +42,9 @@ private: } public: - WeakAndSearchLR(const Terms &terms, uint32_t n) - : _terms(terms, - TermFrequencyScorer(), - 0, - fef::MatchData::UP()), + template<typename Scorer> + WeakAndSearchLR(const Terms &terms, const Scorer & scorer, uint32_t n) + : _terms(terms, scorer, 0, {}), _heaps(DocIdOrder(_terms.docId()), _terms.size()), _algo(), _threshold(1), @@ -105,36 +103,50 @@ WeakAndSearch::visitMembers(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- +template<typename Scorer> SearchIterator::UP -WeakAndSearch::createArrayWand(const Terms &terms, uint32_t n, bool strict) +WeakAndSearch::createArrayWand(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict) { if (strict) { - return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, true>>(terms, n); + return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, true>>(terms, scorer, n); } else { - return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, false>>(terms, n); + return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, false>>(terms, scorer, n); } } +template<typename Scorer> SearchIterator::UP -WeakAndSearch::createHeapWand(const Terms &terms, uint32_t n, bool strict) +WeakAndSearch::createHeapWand(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict) { if (strict) { - return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, true>>(terms, n); + return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, true>>(terms, scorer, n); } else { - return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, false>>(terms, n); + return std::make_unique<wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, false>>(terms, scorer, n); } } +template<typename Scorer> SearchIterator::UP -WeakAndSearch::create(const Terms &terms, uint32_t n, bool strict) +WeakAndSearch::create(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict) { if (terms.size() < 128) { - return createArrayWand(terms, n, strict); + return createArrayWand(terms, scorer, n, strict); } else { - return createHeapWand(terms, n, strict); + return createHeapWand(terms, scorer, n, strict); } } +SearchIterator::UP +WeakAndSearch::create(const Terms &terms, uint32_t n, bool strict) +{ + return create(terms, wand::TermFrequencyScorer(), n, strict); +} + //----------------------------------------------------------------------------- +template SearchIterator::UP WeakAndSearch::create<wand::TermFrequencyScorer>(const Terms &terms, const wand::TermFrequencyScorer & scorer, uint32_t n, bool strict); +template SearchIterator::UP WeakAndSearch::create<wand::Bm25TermFrequencyScorer>(const Terms &terms, const wand::Bm25TermFrequencyScorer & scorer, uint32_t n, bool strict); +template SearchIterator::UP WeakAndSearch::createArrayWand<wand::TermFrequencyScorer>(const Terms &terms, const wand::TermFrequencyScorer & scorer, uint32_t n, bool strict); +template SearchIterator::UP WeakAndSearch::createHeapWand<wand::TermFrequencyScorer>(const Terms &terms, const wand::TermFrequencyScorer & scorer, uint32_t n, bool strict); + } diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h index 6a56a04887c..a91b2860a63 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h @@ -15,8 +15,12 @@ struct WeakAndSearch : SearchIterator { virtual const Terms &getTerms() const = 0; virtual uint32_t getN() const = 0; void visitMembers(vespalib::ObjectVisitor &visitor) const override; - static SearchIterator::UP createArrayWand(const Terms &terms, uint32_t n, bool strict); - static SearchIterator::UP createHeapWand(const Terms &terms, uint32_t n, bool strict); + template<typename Scorer> + static SearchIterator::UP createArrayWand(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict); + template<typename Scorer> + static SearchIterator::UP createHeapWand(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict); + template<typename Scorer> + static SearchIterator::UP create(const Terms &terms, const Scorer & scorer, uint32_t n, bool strict); static SearchIterator::UP create(const Terms &terms, uint32_t n, bool strict); }; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 1db688156e0..b542c422f50 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -672,15 +672,18 @@ HnswIndex<type>::mutual_reconnect(const LinkArrayRef &cluster, uint32_t level) std::vector<PairDist> pairs; for (uint32_t i = 0; i + 1 < cluster.size(); ++i) { uint32_t n_id_1 = cluster[i]; + TypedCells n_cells_1 = get_vector(n_id_1); + if (n_cells_1.non_existing_attribute_value()) [[unlikely]] continue; LinkArrayRef n_list_1 = _graph.get_link_array(n_id_1, level); - std::unique_ptr<BoundDistanceFunction> df; + std::unique_ptr<BoundDistanceFunction> df = _distance_ff->for_insertion_vector(n_cells_1); for (uint32_t j = i + 1; j < cluster.size(); ++j) { uint32_t n_id_2 = cluster[j]; - if (has_link_to(n_list_1, n_id_2)) continue; - if (!df) { - df = _distance_ff->for_insertion_vector(get_vector(n_id_1)); + if ( ! has_link_to(n_list_1, n_id_2)) { + auto n_cells_2 = get_vector(n_id_2); + if (!n_cells_2.non_existing_attribute_value()) { + pairs.emplace_back(n_id_1, n_id_2, df->calc(n_cells_2)); + } } - pairs.emplace_back(n_id_1, n_id_2, calc_distance(*df, n_id_2)); } } std::sort(pairs.begin(), pairs.end()); @@ -1120,6 +1123,32 @@ HnswIndex<type>::count_reachable_nodes() const return {found_cnt, true}; } +template <HnswIndexType type> +uint32_t +HnswIndex<type>::get_subspaces(uint32_t docid) const noexcept +{ + if constexpr (type == HnswIndexType::SINGLE) { + return (docid < _graph.nodes.get_size() && _graph.nodes.get_elem_ref(docid).levels_ref().load_relaxed().valid()) ? 1 : 0; + } else { + return _id_mapping.get_ids(docid).size(); + } +} + +template <HnswIndexType type> +uint32_t +HnswIndex<type>::check_consistency(uint32_t docid_limit) const noexcept +{ + uint32_t inconsistencies = 0; + for (uint32_t docid = 1; docid < docid_limit; ++docid) { + auto index_subspaces = get_subspaces(docid); + auto store_subspaces = get_vectors(docid).subspaces(); + if (index_subspaces != store_subspaces) { + ++inconsistencies; + } + } + return inconsistencies; +} + template class HnswIndex<HnswIndexType::SINGLE>; template class HnswIndex<HnswIndexType::MULTI>; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 616140f426f..4d4440c1bcb 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -193,6 +193,9 @@ protected: LinkArray filter_valid_nodeids(uint32_t level, const internal::PreparedAddNode::Links &neighbors, uint32_t self_nodeid); void internal_complete_add(uint32_t docid, internal::PreparedAddDoc &op); void internal_complete_add_node(uint32_t nodeid, uint32_t docid, uint32_t subspace, internal::PreparedAddNode &prepared_node); + + // Called from writer only. + uint32_t get_subspaces(uint32_t docid) const noexcept; public: HnswIndex(const DocVectorAccess& vectors, DistanceFunctionFactory::UP distance_ff, RandomLevelGenerator::UP level_generator, const HnswIndexConfig& cfg); @@ -248,6 +251,9 @@ public: uint32_t get_active_nodes() const noexcept { return _graph.get_active_nodes(); } + // Called from writer only. + uint32_t check_consistency(uint32_t docid_limit) const noexcept override; + // Should only be used by unit tests. HnswTestNode get_node(uint32_t nodeid) const; void set_node(uint32_t nodeid, const HnswTestNode &node); diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index 8462ff05eca..c2bbd17ce63 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -114,6 +114,12 @@ public: double distance_threshold) const = 0; virtual DistanceFunctionFactory &distance_function_factory() const = 0; + + /* + * Used when checking consistency during load. + * Called from writer only. + */ + virtual uint32_t check_consistency(uint32_t docid_limit) const noexcept = 0; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index a5d670096ab..9f551166a1d 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -418,18 +418,25 @@ TensorAttribute::complete_set_tensor(DocId docid, const vespalib::eval::Value& t std::unique_ptr<PrepareResult> prepare_result) { if (_index && !prepare_result) { - // The tensor cells are unchanged - if (!_is_dense) { - // but labels might have changed. - EntryRef ref = _tensorStore.store_tensor(tensor); - assert(ref.valid()); - setTensorRef(docid, ref); + VectorBundle vectors(tensor.cells().data, tensor.index().size(), _subspace_type); + if (tensor_cells_are_unchanged(docid, vectors)) { + // The tensor cells are unchanged + if (!_is_dense) { + // but labels might have changed. + EntryRef ref = _tensorStore.store_tensor(tensor); + assert(ref.valid()); + setTensorRef(docid, ref); + } + return; } - return; } internal_set_tensor(docid, tensor); if (_index) { - _index->complete_add_document(docid, std::move(prepare_result)); + if (prepare_result) { + _index->complete_add_document(docid, std::move(prepare_result)); + } else { + _index->add_document(docid); + } } } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp index 28c4099c38b..223c9d7d1f2 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp @@ -322,6 +322,9 @@ TensorAttributeLoader::on_load(vespalib::Executor* executor) if (!load_index()) { return false; } + if (dense_store == nullptr) { + check_consistency(docid_limit); + } } else { build_index(executor, docid_limit); } @@ -329,4 +332,15 @@ TensorAttributeLoader::on_load(vespalib::Executor* executor) return true; } +void +TensorAttributeLoader::check_consistency(uint32_t docid_limit) +{ + auto before = vespalib::steady_clock::now(); + uint32_t inconsistencies = _index->check_consistency(docid_limit); + auto after = vespalib::steady_clock::now(); + double elapsed = vespalib::to_s(after - before); + LOG(info, "%u inconsistencies detected after loading index for attribute %s, (check used %6.3fs)", + inconsistencies, _attr.getName().c_str(), elapsed); +} + } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.h index 6bf68957adc..59baaf0b6dc 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.h @@ -34,6 +34,7 @@ class TensorAttributeLoader { void load_tensor_store(search::attribute::BlobSequenceReader& reader, uint32_t docid_limit); void build_index(vespalib::Executor* executor, uint32_t docid_limit); bool load_index(); + void check_consistency(uint32_t docid_limit); public: TensorAttributeLoader(TensorAttribute& attr, GenerationHandler& generation_handler, RefVector& ref_vector, TensorStore& store, NearestNeighborIndex* index); diff --git a/searchlib/src/vespa/searchlib/util/token_extractor.cpp b/searchlib/src/vespa/searchlib/util/token_extractor.cpp index a78f30afe21..6e1573c4551 100644 --- a/searchlib/src/vespa/searchlib/util/token_extractor.cpp +++ b/searchlib/src/vespa/searchlib/util/token_extractor.cpp @@ -143,8 +143,6 @@ TokenExtractor::extract(std::vector<SpanTerm>& terms, const document::StringFiel { auto tree = StringFieldValue::findTree(trees, SPANTREE_NAME); if (tree == nullptr) { - /* field might not be annotated if match type is exact */ - consider_word(terms, text, Span(0, text.size()), nullptr, doc); return; } for (const Annotation & annotation : *tree) { diff --git a/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp b/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp index 493cbe0ecba..3d92cee601a 100644 --- a/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp +++ b/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp @@ -149,7 +149,7 @@ TEST_F(TokensConverterTest, convert_empty_string) TEST_F(TokensConverterTest, convert_plain_string) { - vespalib::string exp(R"(["Foo Bar Baz"])"); + vespalib::string exp(R"([])"); StringFieldValue plain_string("Foo Bar Baz"); EXPECT_EQ(exp, convert(plain_string)); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java index a4216ee1e41..0d94f719824 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthMonitor.java @@ -30,6 +30,7 @@ class StateV1HealthMonitor implements HealthMonitor { @Override public void close() { periodicExecution.cancel(); + updater.close(); } } diff --git a/storage/src/tests/distributor/updateoperationtest.cpp b/storage/src/tests/distributor/updateoperationtest.cpp index 31ebbe19cbb..e00ce249298 100644 --- a/storage/src/tests/distributor/updateoperationtest.cpp +++ b/storage/src/tests/distributor/updateoperationtest.cpp @@ -49,13 +49,13 @@ struct UpdateOperationTest : Test, DistributorStripeTestUtil { const api::ReturnCode& result = api::ReturnCode()); std::shared_ptr<UpdateOperation> - sendUpdate(const std::string& bucketState, bool create_if_missing = false); + sendUpdate(const std::string& bucketState, bool create_if_missing = false, bool cache_create_flag = false); document::BucketId _bId; }; std::shared_ptr<UpdateOperation> -UpdateOperationTest::sendUpdate(const std::string& bucketState, bool create_if_missing) +UpdateOperationTest::sendUpdate(const std::string& bucketState, bool create_if_missing, bool cache_create_flag) { auto update = std::make_shared<document::DocumentUpdate>( *_repo, *_html_type, @@ -67,6 +67,9 @@ UpdateOperationTest::sendUpdate(const std::string& bucketState, bool create_if_m addNodesToBucketDB(_bId, bucketState); auto msg = std::make_shared<api::UpdateCommand>(makeDocumentBucket(document::BucketId(0)), update, 100); + if (cache_create_flag) { + msg->set_cached_create_if_missing(create_if_missing); + } return std::make_shared<UpdateOperation>( node_context(), operation_context(), getDistributorBucketSpace(), msg, std::vector<BucketDatabase::Entry>(), @@ -271,4 +274,20 @@ TEST_F(UpdateOperationTest, cancelled_nodes_are_not_updated_in_db) { dumpBucket(_bId)); } +TEST_F(UpdateOperationTest, cached_create_if_missing_is_propagated_to_fanout_requests) { + setup_stripe(1, 1, "distributor:1 storage:1"); + for (bool cache_flag : {false, true}) { + for (bool create_if_missing : {false, true}) { + std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3", create_if_missing, cache_flag)); + DistributorMessageSenderStub sender; + cb->start(sender); + + ASSERT_EQ("Update => 0", sender.getCommands(true)); + auto& cmd = dynamic_cast<api::UpdateCommand&>(*sender.command(0)); + EXPECT_EQ(cmd.has_cached_create_if_missing(), cache_flag); + EXPECT_EQ(cmd.create_if_missing(), create_if_missing); + } + } +} + } diff --git a/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp b/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp index 698d8dee573..231f41ffd21 100644 --- a/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp +++ b/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp @@ -262,7 +262,6 @@ TEST_P(StorageProtocolTest, response_metadata_is_propagated) { TEST_P(StorageProtocolTest, update) { auto update = std::make_shared<document::DocumentUpdate>(_docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId()); update->addUpdate(FieldUpdate(_testDoc->getField("headerval")).addUpdate(std::make_unique<AssignValueUpdate>(std::make_unique<IntFieldValue>(17)))); - update->addFieldPathUpdate(std::make_unique<RemoveFieldPathUpdate>("headerval", "testdoctype1.headerval > 0")); auto cmd = std::make_shared<UpdateCommand>(_bucket, update, 14); @@ -284,6 +283,37 @@ TEST_P(StorageProtocolTest, update) { EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); } +TEST_P(StorageProtocolTest, update_request_create_if_missing_flag_is_propagated) { + auto make_update_cmd = [&](bool create_if_missing, bool cached) { + auto update = std::make_shared<document::DocumentUpdate>( + _docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId()); + update->addUpdate(FieldUpdate(_testDoc->getField("headerval")).addUpdate( + std::make_unique<AssignValueUpdate>(std::make_unique<IntFieldValue>(17)))); + update->addFieldPathUpdate(std::make_unique<RemoveFieldPathUpdate>("headerval", "testdoctype1.headerval > 0")); + update->setCreateIfNonExistent(create_if_missing); + auto cmd = std::make_shared<UpdateCommand>(_bucket, update, 14); + if (cached) { + cmd->set_cached_create_if_missing(create_if_missing); + } + return cmd; + }; + + auto check_flag_propagation = [&](bool create_if_missing, bool cached) { + auto cmd = make_update_cmd(create_if_missing, cached); + EXPECT_EQ(cmd->has_cached_create_if_missing(), cached); + EXPECT_EQ(cmd->create_if_missing(), create_if_missing); + + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(cmd2->has_cached_create_if_missing(), cached); + EXPECT_EQ(cmd2->create_if_missing(), create_if_missing); + }; + + check_flag_propagation(false, false); + check_flag_propagation(true, false); + check_flag_propagation(false, true); + check_flag_propagation(true, true); +} + TEST_P(StorageProtocolTest, get) { auto cmd = std::make_shared<GetCommand>(_bucket, _testDocId, "foo,bar,vekterli", 123); auto cmd2 = copyCommand(cmd); @@ -880,7 +910,7 @@ TEST_P(StorageProtocolTest, track_memory_footprint_for_some_messages) { EXPECT_EQ(sizeof(BucketInfoCommand), sizeof(BucketCommand)); EXPECT_EQ(sizeof(TestAndSetCommand), sizeof(BucketInfoCommand) + sizeof(vespalib::string)); EXPECT_EQ(sizeof(PutCommand), sizeof(TestAndSetCommand) + 40); - EXPECT_EQ(sizeof(UpdateCommand), sizeof(TestAndSetCommand) + 32); + EXPECT_EQ(sizeof(UpdateCommand), sizeof(TestAndSetCommand) + 40); EXPECT_EQ(sizeof(RemoveCommand), sizeof(TestAndSetCommand) + 112); EXPECT_EQ(sizeof(GetCommand), sizeof(BucketInfoCommand) + sizeof(TestAndSetCondition) + 184); } diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index eb4789b25d4..1eb6bf5dd9a 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -159,28 +159,46 @@ TEST_F(DocumentApiConverterTest, forwarded_put) { } TEST_F(DocumentApiConverterTest, update) { - auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, defaultDocId); - documentapi::UpdateDocumentMessage updateMsg(update); - updateMsg.setOldTimestamp(1234); - updateMsg.setNewTimestamp(5678); - updateMsg.setCondition(my_condition); - - auto updateCmd = toStorageAPI<api::UpdateCommand>(updateMsg); - EXPECT_EQ(defaultBucket, updateCmd->getBucket()); - ASSERT_EQ(update.get(), updateCmd->getUpdate().get()); - EXPECT_EQ(api::Timestamp(1234), updateCmd->getOldTimestamp()); - EXPECT_EQ(api::Timestamp(5678), updateCmd->getTimestamp()); - EXPECT_EQ(my_condition, updateCmd->getCondition()); - - auto mbusReply = updateMsg.createReply(); - ASSERT_TRUE(mbusReply.get()); - toStorageAPI<api::UpdateReply>(*mbusReply, *updateCmd); - - auto mbusUpdate = toDocumentAPI<documentapi::UpdateDocumentMessage>(*updateCmd); - ASSERT_EQ((&mbusUpdate->getDocumentUpdate()), update.get()); - EXPECT_EQ(api::Timestamp(1234), mbusUpdate->getOldTimestamp()); - EXPECT_EQ(api::Timestamp(5678), mbusUpdate->getNewTimestamp()); - EXPECT_EQ(my_condition, mbusUpdate->getCondition()); + auto do_test_update = [&](bool create_if_missing) { + auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, defaultDocId); + update->setCreateIfNonExistent(create_if_missing); + documentapi::UpdateDocumentMessage updateMsg(update); + updateMsg.setOldTimestamp(1234); + updateMsg.setNewTimestamp(5678); + updateMsg.setCondition(my_condition); + EXPECT_FALSE(updateMsg.has_cached_create_if_missing()); + EXPECT_EQ(updateMsg.create_if_missing(), create_if_missing); + + auto updateCmd = toStorageAPI<api::UpdateCommand>(updateMsg); + EXPECT_EQ(defaultBucket, updateCmd->getBucket()); + ASSERT_EQ(update.get(), updateCmd->getUpdate().get()); + EXPECT_EQ(api::Timestamp(1234), updateCmd->getOldTimestamp()); + EXPECT_EQ(api::Timestamp(5678), updateCmd->getTimestamp()); + EXPECT_EQ(my_condition, updateCmd->getCondition()); + EXPECT_FALSE(updateCmd->has_cached_create_if_missing()); + EXPECT_EQ(updateCmd->create_if_missing(), create_if_missing); + + auto mbusReply = updateMsg.createReply(); + ASSERT_TRUE(mbusReply.get()); + toStorageAPI<api::UpdateReply>(*mbusReply, *updateCmd); + + auto mbusUpdate = toDocumentAPI<documentapi::UpdateDocumentMessage>(*updateCmd); + ASSERT_EQ((&mbusUpdate->getDocumentUpdate()), update.get()); + EXPECT_EQ(api::Timestamp(1234), mbusUpdate->getOldTimestamp()); + EXPECT_EQ(api::Timestamp(5678), mbusUpdate->getNewTimestamp()); + EXPECT_EQ(my_condition, mbusUpdate->getCondition()); + EXPECT_EQ(mbusUpdate->create_if_missing(), create_if_missing); + + // Cached value of create_if_missing should override underlying update's value + updateCmd->set_cached_create_if_missing(!create_if_missing); + EXPECT_TRUE(updateCmd->has_cached_create_if_missing()); + EXPECT_EQ(updateCmd->create_if_missing(), !create_if_missing); + mbusUpdate = toDocumentAPI<documentapi::UpdateDocumentMessage>(*updateCmd); + EXPECT_TRUE(mbusUpdate->has_cached_create_if_missing()); + EXPECT_EQ(mbusUpdate->create_if_missing(), !create_if_missing); + }; + do_test_update(false); + do_test_update(true); } TEST_F(DocumentApiConverterTest, remove) { diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 84e9ab71bcb..849746416d6 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -668,7 +668,7 @@ TwoPhaseUpdateOperation::applyUpdateToDocument(document::Document& doc) const bool TwoPhaseUpdateOperation::shouldCreateIfNonExistent() const { - return _updateCmd->getUpdate()->getCreateIfNonExistent(); + return _updateCmd->create_if_missing(); } bool diff --git a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp index 7b6833cc299..2b47d53363f 100644 --- a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp @@ -29,7 +29,7 @@ UpdateOperation::UpdateOperation(const DistributorNodeContext& node_ctx, _msg(msg), _entries(std::move(entries)), _new_timestamp(_msg->getTimestamp()), - _is_auto_create_update(_msg->getUpdate()->getCreateIfNonExistent()), + _is_auto_create_update(_msg->create_if_missing()), _node_ctx(node_ctx), _op_ctx(op_ctx), _bucketSpace(bucketSpace), @@ -112,6 +112,9 @@ UpdateOperation::onStart(DistributorStripeMessageSender& sender) copyMessageSettings(*_msg, *command); command->setOldTimestamp(_msg->getOldTimestamp()); command->setCondition(_msg->getCondition()); + if (_msg->has_cached_create_if_missing()) { + command->set_cached_create_if_missing(_msg->create_if_missing()); + } messages.emplace_back(std::move(command), node); } diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index ca46e87285b..5b8052a05f8 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -54,6 +54,9 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) auto to = std::make_unique<api::UpdateCommand>(bucket, from.stealDocumentUpdate(), from.getNewTimestamp()); to->setOldTimestamp(from.getOldTimestamp()); to->setCondition(from.getCondition()); + if (from.has_cached_create_if_missing()) { + to->set_cached_create_if_missing(from.create_if_missing()); + } toMsg = std::move(to); break; } @@ -217,6 +220,9 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg) to->setOldTimestamp(from.getOldTimestamp()); to->setNewTimestamp(from.getTimestamp()); to->setCondition(from.getCondition()); + if (from.has_cached_create_if_missing()) { + to->set_cached_create_if_missing(from.create_if_missing()); + } toMsg = std::move(to); break; } diff --git a/storage/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storage/src/vespa/storageapi/mbusprot/protobuf/feed.proto index 55d516a017b..403752b0c84 100644 --- a/storage/src/vespa/storageapi/mbusprot/protobuf/feed.proto +++ b/storage/src/vespa/storageapi/mbusprot/protobuf/feed.proto @@ -31,11 +31,18 @@ message Update { } message UpdateRequest { - Bucket bucket = 1; - Update update = 2; - uint64 new_timestamp = 3; - uint64 expected_old_timestamp = 4; // If zero; no expectation. - TestAndSetCondition condition = 5; + enum CreateIfMissing { + UNSPECIFIED = 0; // Legacy fallback: must deserialize `update` to find flag value + TRUE = 1; + FALSE = 2; + } + + Bucket bucket = 1; + Update update = 2; + uint64 new_timestamp = 3; + uint64 expected_old_timestamp = 4; // If zero; no expectation. + TestAndSetCondition condition = 5; + CreateIfMissing create_if_missing = 6; } message UpdateResponse { diff --git a/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 57047be6037..0f4a34cc775 100644 --- a/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -465,6 +465,10 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::UpdateCommand& msg) if (msg.getCondition().isPresent()) { set_tas_condition(*req.mutable_condition(), msg.getCondition()); } + if (msg.has_cached_create_if_missing()) { + req.set_create_if_missing(msg.create_if_missing() ? protobuf::UpdateRequest_CreateIfMissing_TRUE + : protobuf::UpdateRequest_CreateIfMissing_FALSE); + } }); } @@ -482,6 +486,9 @@ api::StorageCommand::UP ProtocolSerialization7::onDecodeUpdateCommand(BBuf& buf) if (req.has_condition()) { cmd->setCondition(get_tas_condition(req.condition())); } + if (req.create_if_missing() != protobuf::UpdateRequest_CreateIfMissing_UNSPECIFIED) { + cmd->set_cached_create_if_missing(req.create_if_missing() == protobuf::UpdateRequest_CreateIfMissing_TRUE); + } return cmd; }); } diff --git a/storage/src/vespa/storageapi/message/persistence.cpp b/storage/src/vespa/storageapi/message/persistence.cpp index 4c24bb74faf..af054855bbe 100644 --- a/storage/src/vespa/storageapi/message/persistence.cpp +++ b/storage/src/vespa/storageapi/message/persistence.cpp @@ -105,13 +105,23 @@ UpdateCommand::UpdateCommand(const document::Bucket &bucket, const document::Doc : TestAndSetCommand(MessageType::UPDATE, bucket), _update(update), _timestamp(time), - _oldTimestamp(0) + _oldTimestamp(0), + _create_if_missing() { if ( ! _update) { throw vespalib::IllegalArgumentException("Cannot update a null update", VESPA_STRLOC); } } +bool +UpdateCommand::create_if_missing() const +{ + if (_create_if_missing.has_value()) { + return *_create_if_missing; + } + return _update->getCreateIfNonExistent(); +} + const document::DocumentType * UpdateCommand::getDocumentType() const { return &_update->getType(); diff --git a/storage/src/vespa/storageapi/message/persistence.h b/storage/src/vespa/storageapi/message/persistence.h index f44ab4e8280..0676e1d0f44 100644 --- a/storage/src/vespa/storageapi/message/persistence.h +++ b/storage/src/vespa/storageapi/message/persistence.h @@ -1,8 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * @file persistence.h - * - * Persistence related commands, like put, get & remove + * Persistence related commands, like put, get & remove */ #pragma once @@ -10,6 +8,7 @@ #include <vespa/storageapi/defs.h> #include <vespa/document/base/documentid.h> #include <vespa/documentapi/messagebus/messages/testandsetcondition.h> +#include <optional> namespace document { class DocumentUpdate; @@ -117,20 +116,32 @@ class UpdateCommand : public TestAndSetCommand { std::shared_ptr<document::DocumentUpdate> _update; Timestamp _timestamp; Timestamp _oldTimestamp; + std::optional<bool> _create_if_missing; // caches the value held (possibly lazily deserialized) in _update public: UpdateCommand(const document::Bucket &bucket, const std::shared_ptr<document::DocumentUpdate>&, Timestamp); ~UpdateCommand() override; - void setTimestamp(Timestamp ts) { _timestamp = ts; } - void setOldTimestamp(Timestamp ts) { _oldTimestamp = ts; } + void setTimestamp(Timestamp ts) noexcept { _timestamp = ts; } + void setOldTimestamp(Timestamp ts) noexcept { _oldTimestamp = ts; } + + [[nodiscard]] bool has_cached_create_if_missing() const noexcept { + return _create_if_missing.has_value(); + } + // It is the caller's responsibility to ensure this value matches that of _update->getCreateIfNonExisting() + void set_cached_create_if_missing(bool create) noexcept { + _create_if_missing = create; + } const std::shared_ptr<document::DocumentUpdate>& getUpdate() const { return _update; } const document::DocumentId& getDocumentId() const override; Timestamp getTimestamp() const { return _timestamp; } Timestamp getOldTimestamp() const { return _oldTimestamp; } + // May throw iff has_cached_create_if_missing() == false, otherwise noexcept. + [[nodiscard]] bool create_if_missing() const; + const document::DocumentType * getDocumentType() const override; vespalib::string getSummary() const override; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index 2c8908a89a6..2f344004780 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -86,6 +86,10 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde return createIdentitySslContext(keyManager, trustStoreFile, false); } + public SSLContext createIdentitySslContextWithTrustStore(Path trustStoreFile, boolean includeDefaultTruststore) { + return createIdentitySslContext(keyManager, trustStoreFile, includeDefaultTruststore); + } + /** * Create an SSL context with the given trust store and the key manager from this provider. * If the {code includeDefaultTruststore} is true, the default trust store will be included. diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java index af0da93edc3..56e64b2261d 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -1,10 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.utils; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzService; import java.io.IOException; import java.io.UncheckedIOException; @@ -132,7 +132,7 @@ public class SiaUtils { try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(keysDirectory)) { return StreamSupport.stream(directoryStream.spliterator(), false) .map(path -> path.getFileName().toString()) - .filter(fileName -> fileName.endsWith(keyFileSuffix)) + .filter(fileName -> fileName.endsWith(keyFileSuffix) && ! fileName.contains(":role.")) .map(fileName -> fileName.substring(0, fileName.length() - keyFileSuffix.length())) .map(AthenzService::new) .collect(toList()); diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java index 9ff59236c0c..8274fe7f7a6 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java @@ -32,6 +32,7 @@ public class SiaUtilsTest { Files.createFile(SiaUtils.getPrivateKeyFile(siaRoot, fooService)); AthenzService barService = new AthenzService("my.domain.bar"); Files.createFile(SiaUtils.getPrivateKeyFile(siaRoot, barService)); + Files.createFile(siaRoot.resolve("keys/my.domain.foo:role.my-role.key.pem")); List<AthenzIdentity> siaIdentities = SiaUtils.findSiaServices(siaRoot); assertEquals(2, siaIdentities.size()); diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index f8966d4fc68..7519f3a5211 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -123,9 +123,7 @@ org.apache.opennlp:opennlp-tools:${opennlp.vespa.version} org.apache.velocity:velocity-engine-core:${velocity.vespa.version} org.apache.yetus:audience-annotations:0.12.0 org.apache.zookeeper:zookeeper-jute:${zookeeper.client.vespa.version} -org.apache.zookeeper:zookeeper-jute:3.9.1 org.apache.zookeeper:zookeeper:${zookeeper.client.vespa.version} -org.apache.zookeeper:zookeeper:3.9.1 org.apiguardian:apiguardian-api:${apiguardian.vespa.version} org.assertj:assertj-core:${assertj.vespa.version} org.bouncycastle:bcpkix-jdk18on:${bouncycastle.vespa.version} diff --git a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java index 11fb6526210..3111815b332 100644 --- a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java +++ b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/JsonFeeder.java @@ -414,7 +414,7 @@ public class JsonFeeder implements Closeable { abstract String getDocumentJson(long start, long end); OperationParseException parseException(String error) { - JsonLocation location = parser.getTokenLocation(); + JsonLocation location = parser.currentLocation(); return new OperationParseException(error + " at offset " + location.getByteOffset() + " (line " + location.getLineNr() + ", column " + location.getColumnNr() + ")"); } @@ -444,13 +444,13 @@ public class JsonFeeder implements Closeable { case "create": parameters = parameters.createIfNonExistent(readBoolean()); break; case "fields": { expect(START_OBJECT); - start = parser.getTokenLocation().getByteOffset(); + start = parser.currentTokenLocation().getByteOffset(); int depth = 1; while (depth > 0) switch (parser.nextToken()) { case START_OBJECT: ++depth; break; case END_OBJECT: --depth; break; } - end = parser.getTokenLocation().getByteOffset() + 1; + end = parser.currentTokenLocation().getByteOffset() + 1; break; } default: throw parseException("Unexpected field name '" + parser.getText() + "'"); @@ -470,7 +470,7 @@ public class JsonFeeder implements Closeable { if (end >= start) throw parseException("Illegal 'fields' object for remove operation"); else - start = end = parser.getTokenLocation().getByteOffset(); // getDocumentJson advances buffer overwrite head. + start = end = parser.currentTokenLocation().getByteOffset(); // getDocumentJson advances buffer overwrite head. } else if (end < start) throw parseException("No 'fields' object for document"); @@ -486,14 +486,14 @@ public class JsonFeeder implements Closeable { private void expect(JsonToken token) throws IOException { if (parser.nextToken() != token) - throw new OperationParseException("Expected '" + token + "' at offset " + parser.getTokenLocation().getByteOffset() + + throw new OperationParseException("Expected '" + token + "' at offset " + parser.currentTokenLocation().getByteOffset() + ", but found '" + parser.currentToken() + "' (" + parser.getText() + ")"); } private String readString() throws IOException { String value = parser.nextTextValue(); if (value == null) - throw new OperationParseException("Expected '" + JsonToken.VALUE_STRING + "' at offset " + parser.getTokenLocation().getByteOffset() + + throw new OperationParseException("Expected '" + JsonToken.VALUE_STRING + "' at offset " + parser.currentTokenLocation().getByteOffset() + ", but found '" + parser.currentToken() + "' (" + parser.getText() + ")"); return value; @@ -502,7 +502,7 @@ public class JsonFeeder implements Closeable { private boolean readBoolean() throws IOException { Boolean value = parser.nextBooleanValue(); if (value == null) - throw new OperationParseException("Expected '" + JsonToken.VALUE_FALSE + "' or '" + JsonToken.VALUE_TRUE + "' at offset " + parser.getTokenLocation().getByteOffset() + + throw new OperationParseException("Expected '" + JsonToken.VALUE_FALSE + "' or '" + JsonToken.VALUE_TRUE + "' at offset " + parser.currentTokenLocation().getByteOffset() + ", but found '" + parser.currentToken() + "' (" + parser.getText() + ")"); return value; diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java index a30cfd5ec39..9dd11113c0b 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java @@ -219,13 +219,13 @@ class HttpFeedClient implements FeedClient { throw new ResultParseException(documentId, "Expected 'trace' to be an array, but got '" + parser.currentToken() + "' in: " + new String(json, UTF_8)); - int start = (int) parser.getTokenLocation().getByteOffset(); + int start = (int) parser.currentTokenLocation().getByteOffset(); int depth = 1; while (depth > 0) switch (parser.nextToken()) { case START_ARRAY: ++depth; break; case END_ARRAY: --depth; break; } - int end = (int) parser.getTokenLocation().getByteOffset() + 1; + int end = (int) parser.currentTokenLocation().getByteOffset() + 1; trace = new String(json, start, end - start, UTF_8); break; default: diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index b483d6977d6..f1829a1c42e 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -1397,6 +1397,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { Phaser phaser = new Phaser(2); // Synchronize this thread (dispatch) with the visitor callback thread. AtomicReference<String> error = new AtomicReference<>(); // Set if error occurs during processing of visited documents. callback.onStart(response, fullyApplied); + final AtomicLong receivedDocsCount = new AtomicLong(0); VisitorControlHandler controller = new VisitorControlHandler() { final ScheduledFuture<?> abort = streaming ? visitDispatcher.schedule(this::abort, visitTimeout(request), MILLISECONDS) : null; final AtomicReference<VisitorSession> session = new AtomicReference<>(); @@ -1410,7 +1411,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { try (response) { callback.onEnd(response); - response.writeDocumentCount(getVisitorStatistics() == null ? 0 : getVisitorStatistics().getDocumentsVisited()); + response.writeDocumentCount(receivedDocsCount.get()); if (session.get() != null) response.writeTrace(session.get().getTrace()); @@ -1456,6 +1457,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { if (m instanceof PutDocumentMessage put) document = put.getDocumentPut().getDocument(); else if (parameters.visitRemoves() && m instanceof RemoveDocumentMessage remove) removeId = remove.getDocumentId(); else throw new UnsupportedOperationException("Got unsupported message type: " + m.getClass().getName()); + receivedDocsCount.getAndAdd(1); callback.onDocument(response, document, removeId, diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 58cf34712aa..3a8456d213d 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -290,7 +290,7 @@ public class DocumentV1ApiTest { parameters.getLocalDataHandler().onMessage(new RemoveDocumentMessage(new DocumentId("id:space:music::t-square-truth")), tokens.get(3)); VisitorStatistics statistics = new VisitorStatistics(); statistics.setBucketsVisited(1); - statistics.setDocumentsVisited(3); + statistics.setDocumentsVisited(123); // Ignored in favor of tracking actually emitted entries parameters.getControlHandler().onVisitorStatistics(statistics); parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.TIMEOUT, "timeout is OK"); }); @@ -323,7 +323,7 @@ public class DocumentV1ApiTest { "remove": "id:space:music::t-square-truth" } ], - "documentCount": 3, + "documentCount": 4, "trace": [ { "message": "Tracy Chapman" }, { @@ -488,7 +488,7 @@ public class DocumentV1ApiTest { assertSameJson(""" { "pathId": "/document/v1/space/music/docid", - "documentCount": 0 + "documentCount": 1 }""", response.readAll()); assertEquals(200, response.getStatus()); @@ -542,7 +542,7 @@ public class DocumentV1ApiTest { assertSameJson(""" { "pathId": "/document/v1/space/music/docid", - "documentCount": 0, + "documentCount": 1, "message": "boom" }""", response.readAll()); diff --git a/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java b/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java index ee755a44010..cccc9667e11 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java +++ b/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java @@ -43,7 +43,7 @@ public class JsonBenchmark { try (JsonParser jsonParser = jsonFactory.createParser(json)) { JsonToken array = jsonParser.nextToken(); for (JsonToken token = jsonParser.nextToken(); !JsonToken.END_ARRAY.equals(token); token = jsonParser.nextToken()) { - if (JsonToken.FIELD_NAME.equals(token) && "weight".equals(jsonParser.getCurrentName())) { + if (JsonToken.FIELD_NAME.equals(token) && "weight".equals(jsonParser.currentName())) { token = jsonParser.nextToken(); count += jsonParser.getLongValue(); } diff --git a/vespalog/src/logger/runserver.cpp b/vespalog/src/logger/runserver.cpp index 9a0a499cd54..4e0141f88dc 100644 --- a/vespalog/src/logger/runserver.cpp +++ b/vespalog/src/logger/runserver.cpp @@ -6,7 +6,7 @@ #include <cerrno> #include <unistd.h> #include <csignal> - +#include <poll.h> #include <sys/select.h> #include <sys/types.h> #include <sys/wait.h> @@ -18,6 +18,7 @@ #include "llreader.h" #include <vespa/log/log.h> #include <chrono> +#include <array> LOG_SETUP("runserver"); @@ -179,8 +180,6 @@ int loop(const char *svc, char * const * run) pstdout[0], pstdout[1], pstderr[0], pstderr[1]); - int high = 1 + pstdout[0] + pstderr[0]; - pid_t child = fork(); if (child == 0) { @@ -237,24 +236,24 @@ int loop(const char *svc, char * const * run) bool outeof = false; bool erreof = false; - + constexpr int stdout_idx = 0, stderr_idx = 1; + std::array<pollfd, 2> fds{}; int wstat = 0; while (child || !outeof || !erreof) { - struct timeval timeout; - - timeout.tv_sec = 0; - timeout.tv_usec = 100000; // == 100 ms == 1/10 s - - fd_set pipes; - - FD_ZERO(&pipes); - if (!outeof) FD_SET(pstdout[0], &pipes); - if (!erreof) FD_SET(pstderr[0], &pipes); - - int n = select(high, &pipes, NULL, NULL, &timeout); + // Entries with negative fds are entirely ignored by the kernel. + fds[stdout_idx].fd = !outeof ? pstdout[0] : -1; + fds[stdout_idx].events = POLLIN; + fds[stdout_idx].revents = 0; + fds[stderr_idx].fd = !erreof ? pstderr[0] : -2; + fds[stderr_idx].events = POLLIN; + fds[stderr_idx].revents = 0; + + constexpr int poll_timeout_ms = 100; + int n = poll(fds.data(), fds.size(), poll_timeout_ms); if (n > 0) { - if (FD_ISSET(pstdout[0], &pipes)) { + constexpr short ev_mask = POLLIN | POLLERR | POLLHUP; + if ((fds[stdout_idx].revents & ev_mask) != 0) { LOG(debug, "out reader has input"); if (outReader.blockRead()) { while (outReader.hasInput()) { @@ -267,7 +266,7 @@ int loop(const char *svc, char * const * run) close(pstdout[0]); } } - if (FD_ISSET(pstderr[0], &pipes)) { + if ((fds[stderr_idx].revents & ev_mask) != 0) { LOG(debug, "err reader has input"); if (errReader.blockRead()) { while (errReader.hasInput()) { diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp index cee709ed0ed..ba330d14125 100644 --- a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp @@ -58,9 +58,9 @@ MMapPool::mmap(size_t sz) { } buf = ::mmap(nullptr, sz, prot, flags, -1, 0); if (buf == MAP_FAILED) { - fprintf(_G_logFile, "Failed mmaping anonymous of size %ld errno(%d) from : ", sz, errno); + fprintf(_G_logFile, "Will exit due to: Failed mmaping anonymous of size %ld errno(%d) from : ", sz, errno); logStackTrace(); - abort(); + std::quick_exit(66); } } else { if (_has_hugepage_failure_just_happened) { diff --git a/vespamalloc/src/vespamalloc/util/callstack.cpp b/vespamalloc/src/vespamalloc/util/callstack.cpp index b8449c89a72..56b634bca33 100644 --- a/vespamalloc/src/vespamalloc/util/callstack.cpp +++ b/vespamalloc/src/vespamalloc/util/callstack.cpp @@ -1,39 +1,59 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <dlfcn.h> -#include <ctype.h> +#include <cctype> #include <vespamalloc/util/callstack.h> +#include <string> +#include <cxxabi.h> namespace vespamalloc { -const char * -dlAddr(const void * func) { - static const char * _unknown = "UNKNOWN"; - const char * funcName = _unknown; +namespace { + +std::string +demangle(const char *native) { + int status = 0; + size_t size = 0; + char *unmangled = abi::__cxa_demangle(native, nullptr, &size, &status); + if (unmangled == nullptr) { + return ""; // Demangling failed for some reason. TODO return `native` instead? + } + std::string result(unmangled); + free(unmangled); + return result; +} + + +std::string +dlAddr(const void *func) { + static std::string _unknown = "UNKNOWN"; Dl_info info; int ret = dladdr(func, &info); if (ret != 0) { - funcName = info.dli_sname; + return demangle(info.dli_sname); } - return funcName; + return _unknown; +} + } namespace { void verifyAndCopy(const void *addr, char *v, size_t sz) { size_t pos(0); - const char *sym = dlAddr(addr); - for (; sym && (sym[pos] != '\0') && (pos < sz - 1); pos++) { + std::string sym = dlAddr(addr); + for (; (pos < sym.size()) && (pos < sz - 1); pos++) { char c(sym[pos]); v[pos] = isprint(c) ? c : '.'; } v[pos] = '\0'; } + } void StackReturnEntry::info(FILE * os) const { - static char tmp[0x400]; + char tmp[0x400]; verifyAndCopy(_return, tmp, sizeof(tmp)); fprintf(os, "%s(%p)", tmp, _return); } @@ -41,8 +61,8 @@ StackReturnEntry::info(FILE * os) const asciistream & operator << (asciistream & os, const StackReturnEntry & v) { - static char tmp[0x100]; - static char t[0x200]; + char tmp[0x100]; + char t[0x200]; verifyAndCopy(v._return, tmp, sizeof(tmp)); snprintf(t, sizeof(t), "%s(%p)", tmp, v._return); return os << t; diff --git a/vespamalloc/src/vespamalloc/util/callstack.h b/vespamalloc/src/vespamalloc/util/callstack.h index 3773d3c08b2..f3b177ea5f6 100644 --- a/vespamalloc/src/vespamalloc/util/callstack.h +++ b/vespamalloc/src/vespamalloc/util/callstack.h @@ -9,8 +9,6 @@ namespace vespamalloc { -const char * dlAddr(const void * addr); - class StackReturnEntry { public: StackReturnEntry(const void * returnAddress = nullptr, diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyMetrics.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyMetrics.java index 01db658594f..fd6008ac4f9 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyMetrics.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LatencyMetrics.java @@ -59,7 +59,7 @@ public class LatencyMetrics { /** Returns the average number of intervals that ended in the period per second. */ public double endHz() { return roundTo3DecimalPlaces(endHz); } - /** Returns the average load of the implied time periond, for each thread with non-zero load, with 3 decimal places precision. */ + /** Returns the average load of the implied time period, for each thread with non-zero load, with 3 decimal places precision. */ public Map<String, Double> loadByThread() { Map<String, Double> result = new TreeMap<>(); loadByThread.forEach((name, load) -> result.put(name, roundTo3DecimalPlaces(load))); diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index 5e40f1e2246..0fc2eeec46a 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) add_subdirectory(zookeeper-server) -add_subdirectory(zookeeper-server-3.9.2) + diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index 4b7f4be7a7f..e0838a9eefa 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -14,7 +14,6 @@ <modules> <module>zookeeper-server-common</module> <module>zookeeper-server</module> - <module>zookeeper-server-3.9.2</module> </modules> <dependencies> <dependency> diff --git a/zookeeper-server/zookeeper-server-3.9.2/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.9.2/CMakeLists.txt deleted file mode 100644 index de5780610d9..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar(zookeeper-server-3.9.2-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-3.9.2/pom.xml b/zookeeper-server/zookeeper-server-3.9.2/pom.xml deleted file mode 100644 index 791c026234a..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/pom.xml +++ /dev/null @@ -1,99 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server-parent</artifactId> - <version>8-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - <artifactId>zookeeper-server-3.9.2</artifactId> - <packaging>container-plugin</packaging> - <version>8-SNAPSHOT</version> - <properties> - <zookeeper.version>3.9.2</zookeeper.version> - </properties> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server-common</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-client-common</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <!-- Don't use ZK version from zookeeper-client-common --> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>${zookeeper.version}</version> - <exclusions> - <!-- - Container provides wiring for all common log libraries - Duplicate embedding results in various warnings being printed to stderr - --> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <!-- snappy-java and metrics-core are included here - to be able to work with ZooKeeper 3.7.0 due to - class loading issues --> - <dependency> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-core</artifactId> - <scope>compile</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.xerial.snappy</groupId> - <artifactId>snappy-java</artifactId> - <scope>compile</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <!-- Turn off classfile warnings where spotbugs is pulled in transitively. --> - <arg>-Xlint:all</arg> - <arg>-Xlint:-classfile</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - <configuration> - <importPackage>com.sun.management</importPackage> - <bundleSymbolicName>zookeeper-server</bundleSymbolicName> - </configuration> - </plugin> - </plugins> - </build> -</project> diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/ConfigServerZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/ConfigServerZooKeeperServer.java deleted file mode 100644 index a7cd14c415f..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/ConfigServerZooKeeperServer.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; -import com.yahoo.component.annotation.Inject; -import com.yahoo.vespa.zookeeper.server.VespaZooKeeperServer; - -import java.nio.file.Path; - -/** - * - * Server used for starting config server, needed to be able to have different behavior for hosted and - * self-hosted Vespa (controlled by zookeeperServerConfig.dynamicReconfiguration). - * - * @author Harald Musum - */ -public class ConfigServerZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { - - private final VespaZooKeeperServer zooKeeperServer; - - @Inject - public ConfigServerZooKeeperServer(ZookeeperServerConfig zookeeperServerConfig) { - this.zooKeeperServer = zookeeperServerConfig.dynamicReconfiguration() - ? new ReconfigurableVespaZooKeeperServer(new Reconfigurer(new VespaZooKeeperAdminImpl()), zookeeperServerConfig) - : new VespaZooKeeperServerImpl(zookeeperServerConfig); - } - - @Override - public void deconstruct() { zooKeeperServer.shutdown(); } - - @Override - public void shutdown() { - zooKeeperServer.shutdown(); - } - - @Override - public void start(Path configFilePath) { - zooKeeperServer.start(configFilePath); - } - - @Override - public boolean reconfigurable() { return zooKeeperServer.reconfigurable(); } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java deleted file mode 100644 index d869cbb6938..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import ai.vespa.validation.Validation; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; -import com.yahoo.component.annotation.Inject; -import com.yahoo.vespa.zookeeper.server.VespaZooKeeperServer; - -import java.nio.file.Path; -import java.time.Duration; - -/** - * Starts or reconfigures zookeeper cluster. - * The QuorumPeer conditionally created here is owned by the Reconfigurer; - * when it already has a peer, that peer is used here in case start or shutdown is required. - * Guarantees that server is up by writing a node to ZooKeeper successfully before - * returning from constructor. - * - * @author hmusum - */ -public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { - - private QuorumPeer peer; - - @Inject - public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { - Validation.require(zookeeperServerConfig.dynamicReconfiguration(), - zookeeperServerConfig.dynamicReconfiguration(), - "dynamicReconfiguration must be true"); - peer = reconfigurer.startOrReconfigure(zookeeperServerConfig, this, () -> peer = new VespaQuorumPeer()); - } - - @Override - public void shutdown() { - peer.shutdown(Duration.ofMinutes(1)); - } - - @Override - public void start(Path configFilePath) { - peer.start(configFilePath); - } - - @Override - public boolean reconfigurable() { - return true; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java deleted file mode 100644 index 90554910293..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.security.X509SslContext; -import com.yahoo.security.tls.TlsContext; -import com.yahoo.security.tls.TransportSecurityUtils; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.common.ClientX509Util; -import org.apache.zookeeper.common.X509Exception; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.server.ServerCnxn; -import org.apache.zookeeper.server.auth.AuthenticationProvider; -import org.apache.zookeeper.server.auth.X509AuthenticationProvider; - -import javax.net.ssl.KeyManager; -import javax.net.ssl.X509KeyManager; -import javax.net.ssl.X509TrustManager; -import java.security.cert.X509Certificate; -import java.util.logging.Logger; - -/** - * A {@link AuthenticationProvider} to be used in combination with Vespa mTLS. - * - * @author bjorncs - */ -public class VespaMtlsAuthenticationProvider extends X509AuthenticationProvider { - - private static final Logger log = Logger.getLogger(VespaMtlsAuthenticationProvider.class.getName()); - - public VespaMtlsAuthenticationProvider() { - super(null, null); - } - - @Override - public KeeperException.Code handleAuthentication(ServerCnxn cnxn, byte[] authData) { - // Vespa's mTLS peer authorization rules are performed by the underlying trust manager implementation. - // The client is authorized once the SSL handshake has completed. - X509Certificate[] certificateChain = (X509Certificate[]) cnxn.getClientCertificateChain(); - if (certificateChain == null || certificateChain.length == 0) { - log.warning("Client not authenticated - should not be possible with clientAuth=NEED"); - return KeeperException.Code.AUTHFAILED; - } - X509Certificate certificate = certificateChain[0]; - cnxn.addAuthInfo(new Id(getScheme(), certificate.getSubjectX500Principal().getName())); - return KeeperException.Code.OK; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java deleted file mode 100644 index dd5ac4e252b..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.protect.Process; -import org.apache.zookeeper.server.admin.AdminServer; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.apache.zookeeper.server.quorum.QuorumPeerMain; - -import java.io.IOException; -import java.nio.file.Path; -import java.time.Duration; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Starts or stops a ZooKeeper server. Extends QuorumPeerMain to be able to call initializeAndRun() and wraps - * exceptions so that it can be used by code that does not depend on ZooKeeper. - * - * @author hmusum - */ -class VespaQuorumPeer extends QuorumPeerMain implements QuorumPeer { - - private static final Logger log = java.util.logging.Logger.getLogger(VespaQuorumPeer.class.getName()); - - @Override - public void start(Path path) { - initializeAndRun(new String[]{ path.toFile().getAbsolutePath()}); - } - - @Override - public void shutdown(Duration timeout) { - if (quorumPeer != null) { - log.log(Level.FINE, "Shutting down ZooKeeper server"); - try { - quorumPeer.shutdown(); - quorumPeer.join(timeout.toMillis()); // Wait for shutdown to complete - if (quorumPeer.isAlive()) - throw new IllegalStateException("Peer still alive after " + timeout); - } catch (RuntimeException | InterruptedException e) { - // If shutdown fails, we have no other option than forcing the JVM to stop and letting it be restarted. - // - // When a VespaZooKeeperServer component receives a new config, the container will try to start a new - // server with the new config, this will fail until the old server is deconstructed. If the old server - // fails to deconstruct/shutdown, the new one will never start and if that happens forcing a restart is - // the better option. - Process.logAndDie("Failed to shut down ZooKeeper server properly, forcing shutdown", e); - } - } - } - - @Override - protected void initializeAndRun(String[] args) { - try { - super.initializeAndRun(args); - } catch (QuorumPeerConfig.ConfigException | IOException | AdminServer.AdminServerException e) { - throw new RuntimeException("Exception when initializing or running ZooKeeper server", e); - } - } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java deleted file mode 100644 index c74a020bcf4..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.net.HostName; -import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.admin.ZooKeeperAdmin; -import org.apache.zookeeper.data.ACL; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * @author hmusum - */ -@SuppressWarnings("unused") // Created by injection -public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { - - private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); - - - @SuppressWarnings("try") - @Override - public void reconfigure(String connectionSpec, String servers) throws ReconfigException { - try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(connectionSpec)) { - long fromConfig = -1; - // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0). - log.log(Level.INFO, "Applying ZooKeeper config: " + servers); - byte[] appliedConfig = zooKeeperAdmin.reconfigure(null, null, servers, fromConfig, null); - log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); - - // Verify by issuing a write operation; this is only accepted once new quorum is obtained. - List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; - String node = zooKeeperAdmin.create("/reconfigure-dummy-node", new byte[0], acl, CreateMode.EPHEMERAL_SEQUENTIAL); - zooKeeperAdmin.delete(node, -1); - - log.log(Level.INFO, "Verified ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); - } - catch ( KeeperException.ReconfigInProgress - | KeeperException.ConnectionLossException - | KeeperException.NewConfigNoQuorum e) { - throw new ReconfigException(e); - } - catch (KeeperException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - private ZooKeeperAdmin createAdmin(String connectionSpec) { - return uncheck(() -> new ZooKeeperAdmin(connectionSpec, (int) sessionTimeout().toMillis(), - (event) -> log.log(Level.FINE, event.toString()), new ZkClientConfigBuilder().toConfig())); - } - - /** Creates a node in zookeeper, with hostname as part of node name, this ensures that server is up and working before returning */ - @SuppressWarnings("try") - void createDummyNode(ZookeeperServerConfig zookeeperServerConfig) { - int sleepTime = 2_000; - try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(localConnectionSpec(zookeeperServerConfig))) { - Instant end = Instant.now().plus(Duration.ofMinutes(5)); - Exception exception = null; - do { - try { - zooKeeperAdmin.create("/dummy-node-" + HostName.getLocalhost(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - return; - } catch (KeeperException e) { - if (e instanceof KeeperException.NodeExistsException) { - try { - zooKeeperAdmin.setData("/dummy-node-" + HostName.getLocalhost(), new byte[0], -1); - return; - } catch (KeeperException ex) { - log.log(Level.FINE, e.getMessage()); - Thread.sleep(sleepTime); - continue; - } - } - log.log(Level.FINE, e.getMessage()); - exception = e; - Thread.sleep(sleepTime); - } - } while (Instant.now().isBefore(end)); - throw new RuntimeException("Unable to create dummy node: ", exception); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - -} - diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java deleted file mode 100644 index 4f93eb0efa5..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import ai.vespa.validation.Validation; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; -import com.yahoo.component.annotation.Inject; -import com.yahoo.vespa.zookeeper.server.VespaZooKeeperServer; - -import java.nio.file.Path; -import java.time.Duration; - -/** - * ZooKeeper server. Guarantees that the server is up by writing a node to ZooKeeper successfully before - * returning from constructor. - * - * @author Ulf Lilleengen - * @author Harald Musum - */ -public class VespaZooKeeperServerImpl extends AbstractComponent implements VespaZooKeeperServer { - - private final VespaQuorumPeer peer; - private final ZooKeeperRunner runner; - - @Inject - public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { - Validation.require(! zookeeperServerConfig.dynamicReconfiguration(), - ! zookeeperServerConfig.dynamicReconfiguration(), - "dynamicReconfiguration must be false"); - this.peer = new VespaQuorumPeer(); - this.runner = new ZooKeeperRunner(zookeeperServerConfig, this); - new VespaZooKeeperAdminImpl().createDummyNode(zookeeperServerConfig); - } - - @Override - public void deconstruct() { - runner.shutdown(); - super.deconstruct(); - } - - @Override - public void shutdown() { - peer.shutdown(Duration.ofMinutes(1)); - } - - @Override - public void start(Path configFilePath) { - peer.start(configFilePath); - } - - @Override - public boolean reconfigurable() { - return false; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/common/ClientX509Util.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/common/ClientX509Util.java deleted file mode 100644 index f6dfb0fa4d9..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/common/ClientX509Util.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.common; - -import com.yahoo.vespa.zookeeper.tls.VespaZookeeperTlsContextUtils; -import io.netty.handler.ssl.DelegatingSslContext; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslContextBuilder; -import io.netty.handler.ssl.SslProvider; -import javax.net.ssl.KeyManager; -import javax.net.ssl.SSLEngine; -import javax.net.ssl.SSLException; -import javax.net.ssl.SSLParameters; -import javax.net.ssl.TrustManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -/** - * X509 utilities specific for client-server communication framework. - * <p> - * <em>Modified to use Vespa's TLS context, whenever it is available, instead of the file-based key and trust stores of ZK 3.9. - * Based on https://github.com/apache/zookeeper/blob/branch-3.9/zookeeper-server/src/main/java/org/apache/zookeeper/common/ClientX509Util.java</em> - * - * @author jonmv - */ -public class ClientX509Util extends X509Util { - - private static final Logger LOG = LoggerFactory.getLogger(ClientX509Util.class); - - private final String sslAuthProviderProperty = getConfigPrefix() + "authProvider"; - private final String sslProviderProperty = getConfigPrefix() + "sslProvider"; - - @Override - protected String getConfigPrefix() { - return "zookeeper.ssl."; - } - - @Override - protected boolean shouldVerifyClientHostname() { - return false; - } - - public String getSslAuthProviderProperty() { - return sslAuthProviderProperty; - } - - public String getSslProviderProperty() { - return sslProviderProperty; - } - - public SslContext createNettySslContextForClient(ZKConfig config) - throws X509Exception.KeyManagerException, X509Exception.TrustManagerException, SSLException { - SslContextBuilder sslContextBuilder = SslContextBuilder.forClient(); - KeyManager km; - TrustManager tm; - if (VespaZookeeperTlsContextUtils.tlsContext().isPresent()) { - km = VespaZookeeperTlsContextUtils.tlsContext().get().sslContext().keyManager(); - tm = VespaZookeeperTlsContextUtils.tlsContext().get().sslContext().trustManager(); - } - else { - String keyStoreLocation = config.getProperty(getSslKeystoreLocationProperty(), ""); - String keyStorePassword = getPasswordFromConfigPropertyOrFile(config, getSslKeystorePasswdProperty(), - getSslKeystorePasswdPathProperty()); - String keyStoreType = config.getProperty(getSslKeystoreTypeProperty()); - - if (keyStoreLocation.isEmpty()) { - LOG.warn("{} not specified", getSslKeystoreLocationProperty()); - km = null; - } - else { - km = createKeyManager(keyStoreLocation, keyStorePassword, keyStoreType); - } - - tm = getTrustManager(config); - } - - if (km != null) { - sslContextBuilder.keyManager(km); - } - if (tm != null) { - sslContextBuilder.trustManager(tm); - } - - sslContextBuilder.enableOcsp(config.getBoolean(getSslOcspEnabledProperty())); - sslContextBuilder.protocols(getEnabledProtocols(config)); - Iterable<String> enabledCiphers = getCipherSuites(config); - if (enabledCiphers != null) { - sslContextBuilder.ciphers(enabledCiphers); - } - sslContextBuilder.sslProvider(getSslProvider(config)); - - SslContext sslContext1 = sslContextBuilder.build(); - - if (getFipsMode(config) && isServerHostnameVerificationEnabled(config)) { - return addHostnameVerification(sslContext1, "Server"); - } else { - return sslContext1; - } - } - - public SslContext createNettySslContextForServer(ZKConfig config) - throws X509Exception.SSLContextException, X509Exception.KeyManagerException, X509Exception.TrustManagerException, SSLException { - KeyManager km; - TrustManager tm; - if (VespaZookeeperTlsContextUtils.tlsContext().isPresent()) { - km = VespaZookeeperTlsContextUtils.tlsContext().get().sslContext().keyManager(); - tm = VespaZookeeperTlsContextUtils.tlsContext().get().sslContext().trustManager(); - } - else { - String keyStoreLocation = config.getProperty(getSslKeystoreLocationProperty(), ""); - String keyStorePassword = getPasswordFromConfigPropertyOrFile(config, getSslKeystorePasswdProperty(), - getSslKeystorePasswdPathProperty()); - String keyStoreType = config.getProperty(getSslKeystoreTypeProperty()); - - if (keyStoreLocation.isEmpty()) { - throw new X509Exception.SSLContextException( - "Keystore is required for SSL server: " + getSslKeystoreLocationProperty()); - } - km = createKeyManager(keyStoreLocation, keyStorePassword, keyStoreType); - tm = getTrustManager(config); - } - return createNettySslContextForServer(config, km, tm); - } - - public SslContext createNettySslContextForServer(ZKConfig config, KeyManager keyManager, TrustManager trustManager) throws SSLException { - SslContextBuilder sslContextBuilder = SslContextBuilder.forServer(keyManager); - - if (trustManager != null) { - sslContextBuilder.trustManager(trustManager); - } - - sslContextBuilder.enableOcsp(config.getBoolean(getSslOcspEnabledProperty())); - sslContextBuilder.protocols(getEnabledProtocols(config)); - sslContextBuilder.clientAuth(getClientAuth(config).toNettyClientAuth()); - Iterable<String> enabledCiphers = getCipherSuites(config); - if (enabledCiphers != null) { - sslContextBuilder.ciphers(enabledCiphers); - } - sslContextBuilder.sslProvider(getSslProvider(config)); - - SslContext sslContext1 = sslContextBuilder.build(); - - if (getFipsMode(config) && isClientHostnameVerificationEnabled(config)) { - return addHostnameVerification(sslContext1, "Client"); - } else { - return sslContext1; - } - } - - private SslContext addHostnameVerification(SslContext sslContext, String clientOrServer) { - return new DelegatingSslContext(sslContext) { - @Override - protected void initEngine(SSLEngine sslEngine) { - SSLParameters sslParameters = sslEngine.getSSLParameters(); - sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); - sslEngine.setSSLParameters(sslParameters); - if (LOG.isDebugEnabled()) { - LOG.debug("{} hostname verification: enabled HTTPS style endpoint identification algorithm", clientOrServer); - } - } - }; - } - - private String[] getEnabledProtocols(final ZKConfig config) { - String enabledProtocolsInput = config.getProperty(getSslEnabledProtocolsProperty()); - if (enabledProtocolsInput == null) { - return new String[]{ config.getProperty(getSslProtocolProperty(), DEFAULT_PROTOCOL) }; - } - return enabledProtocolsInput.split(","); - } - - private X509Util.ClientAuth getClientAuth(final ZKConfig config) { - return X509Util.ClientAuth.fromPropertyValue(config.getProperty(getSslClientAuthProperty())); - } - - private Iterable<String> getCipherSuites(final ZKConfig config) { - String cipherSuitesInput = config.getProperty(getSslCipherSuitesProperty()); - if (cipherSuitesInput == null) { - if (getSslProvider(config) != SslProvider.JDK) { - return null; - } - return List.of(X509Util.getDefaultCipherSuites()); - } else { - return List.of(cipherSuitesInput.split(",")); - } - } - - public SslProvider getSslProvider(ZKConfig config) { - return SslProvider.valueOf(config.getProperty(getSslProviderProperty(), "JDK")); - } - - private TrustManager getTrustManager(ZKConfig config) throws X509Exception.TrustManagerException { - String trustStoreLocation = config.getProperty(getSslTruststoreLocationProperty(), ""); - String trustStorePassword = getPasswordFromConfigPropertyOrFile(config, getSslTruststorePasswdProperty(), - getSslTruststorePasswdPathProperty()); - String trustStoreType = config.getProperty(getSslTruststoreTypeProperty()); - - boolean sslCrlEnabled = config.getBoolean(getSslCrlEnabledProperty()); - boolean sslOcspEnabled = config.getBoolean(getSslOcspEnabledProperty()); - boolean sslServerHostnameVerificationEnabled = isServerHostnameVerificationEnabled(config); - boolean sslClientHostnameVerificationEnabled = isClientHostnameVerificationEnabled(config); - - if (trustStoreLocation.isEmpty()) { - LOG.warn("{} not specified", getSslTruststoreLocationProperty()); - return null; - } else { - return createTrustManager(trustStoreLocation, trustStorePassword, trustStoreType, - sslCrlEnabled, sslOcspEnabled, sslServerHostnameVerificationEnabled, - sslClientHostnameVerificationEnabled, getFipsMode(config)); - } - } -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/common/NetUtils.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/common/NetUtils.java deleted file mode 100644 index baa69f12968..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/common/NetUtils.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.common; - -import java.net.Inet6Address; -import java.net.InetAddress; -import java.net.InetSocketAddress; - -/** - * This class contains common utilities for netstuff. Like printing IPv6 literals correctly - */ -public class NetUtils { - - // Note: Changed from original to use hostname from InetSocketAddress if there exists one - public static String formatInetAddr(InetSocketAddress addr) { - String hostName = addr.getHostName(); - if (hostName != null) { - return String.format("%s:%s", hostName, addr.getPort()); - } - - InetAddress ia = addr.getAddress(); - - if (ia == null) { - return String.format("%s:%s", addr.getHostString(), addr.getPort()); - } - if (ia instanceof Inet6Address) { - return String.format("[%s]:%s", ia.getHostAddress(), addr.getPort()); - } else { - return String.format("%s:%s", ia.getHostAddress(), addr.getPort()); - } - } - - /** - * Separates host and port from given host port string if host port string is enclosed - * within square bracket. - * - * @param hostPort host port string - * @return String[]{host, port} if host port string is host:port - * or String[] {host, port:port} if host port string is host:port:port - * or String[] {host} if host port string is host - * or String[]{} if not a ipv6 host port string. - */ - public static String[] getIPV6HostAndPort(String hostPort) { - if (hostPort.startsWith("[")) { - int i = hostPort.lastIndexOf(']'); - if (i < 0) { - throw new IllegalArgumentException( - hostPort + " starts with '[' but has no matching ']'"); - } - String host = hostPort.substring(1, i); - if (host.isEmpty()) { - throw new IllegalArgumentException(host + " is empty."); - } - if (hostPort.length() > i + 1) { - return getHostPort(hostPort, i, host); - } - return new String[] { host }; - } else { - //Not an IPV6 host port string - return new String[] {}; - } - } - - private static String[] getHostPort(String hostPort, int indexOfClosingBracket, String host) { - // [127::1]:2181 , check separator : exits - if (hostPort.charAt(indexOfClosingBracket + 1) != ':') { - throw new IllegalArgumentException(hostPort + " does not have : after ]"); - } - // [127::1]: scenario - if (indexOfClosingBracket + 2 == hostPort.length()) { - throw new IllegalArgumentException(hostPort + " doesn't have a port after colon."); - } - //do not include - String port = hostPort.substring(indexOfClosingBracket + 2); - return new String[] { host, port }; - } -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/SyncRequestProcessor.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/SyncRequestProcessor.java deleted file mode 100644 index cf7f4c44015..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/SyncRequestProcessor.java +++ /dev/null @@ -1,353 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server; - -import java.io.Flushable; -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.Objects; -import java.util.Queue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import org.apache.zookeeper.common.Time; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This RequestProcessor logs requests to disk. It batches the requests to do - * the io efficiently. The request is not passed to the next RequestProcessor - * until its log has been synced to disk. - * - * SyncRequestProcessor is used in 3 different cases - * 1. Leader - Sync request to disk and forward it to AckRequestProcessor which - * send ack back to itself. - * 2. Follower - Sync request to disk and forward request to - * SendAckRequestProcessor which send the packets to leader. - * SendAckRequestProcessor is flushable which allow us to force - * push packets to leader. - * 3. Observer - Sync committed request to disk (received as INFORM packet). - * It never send ack back to the leader, so the nextProcessor will - * be null. This change the semantic of txnlog on the observer - * since it only contains committed txns. - */ -public class SyncRequestProcessor extends ZooKeeperCriticalThread implements RequestProcessor { - - private static final Logger LOG = LoggerFactory.getLogger(SyncRequestProcessor.class); - - private static final Request REQUEST_OF_DEATH = Request.requestOfDeath; - - private static class FlushRequest extends Request { - private final CountDownLatch latch = new CountDownLatch(1); - public FlushRequest() { - super(null, 0, 0, 0, null, null); - } - } - - private static final Request TURN_FORWARDING_DELAY_ON_REQUEST = new Request(null, 0, 0, 0, null, null); - private static final Request TURN_FORWARDING_DELAY_OFF_REQUEST = new Request(null, 0, 0, 0, null, null); - - private static class DelayingProcessor implements RequestProcessor, Flushable { - private final RequestProcessor next; - private Queue<Request> delayed = null; - private DelayingProcessor(RequestProcessor next) { - this.next = next; - } - @Override - public void flush() throws IOException { - if (delayed == null && next instanceof Flushable) { - ((Flushable) next).flush(); - } - } - @Override - public void processRequest(Request request) throws RequestProcessorException { - if (delayed == null) { - next.processRequest(request); - } else { - delayed.add(request); - } - } - @Override - public void shutdown() { - next.shutdown(); - } - private void startDelaying() { - if (delayed == null) { - delayed = new ArrayDeque<>(); - } - } - private void flushAndStopDelaying() throws RequestProcessorException { - if (delayed != null) { - for (Request request : delayed) { - next.processRequest(request); - } - delayed = null; - } - } - } - - /** The number of log entries to log before starting a snapshot */ - private static int snapCount = ZooKeeperServer.getSnapCount(); - - /** - * The total size of log entries before starting a snapshot - */ - private static long snapSizeInBytes = ZooKeeperServer.getSnapSizeInBytes(); - - /** - * Random numbers used to vary snapshot timing - */ - private int randRoll; - private long randSize; - - private final BlockingQueue<Request> queuedRequests = new LinkedBlockingQueue<>(); - - private final Semaphore snapThreadMutex = new Semaphore(1); - - private final ZooKeeperServer zks; - - private final DelayingProcessor nextProcessor; - - /** - * Transactions that have been written and are waiting to be flushed to - * disk. Basically this is the list of SyncItems whose callbacks will be - * invoked after flush returns successfully. - */ - private final Queue<Request> toFlush; - private long lastFlushTime; - - public SyncRequestProcessor(ZooKeeperServer zks, RequestProcessor nextProcessor) { - super("SyncThread:" + zks.getServerId(), zks.getZooKeeperServerListener()); - this.zks = zks; - this.nextProcessor = nextProcessor == null ? null : new DelayingProcessor(nextProcessor); - this.toFlush = new ArrayDeque<>(zks.getMaxBatchSize()); - } - - /** - * used by tests to check for changing - * snapcounts - * @param count - */ - public static void setSnapCount(int count) { - snapCount = count; - } - - /** - * used by tests to get the snapcount - * @return the snapcount - */ - public static int getSnapCount() { - return snapCount; - } - - private long getRemainingDelay() { - long flushDelay = zks.getFlushDelay(); - long duration = Time.currentElapsedTime() - lastFlushTime; - if (duration < flushDelay) { - return flushDelay - duration; - } - return 0; - } - - /** If both flushDelay and maxMaxBatchSize are set (bigger than 0), flush - * whenever either condition is hit. If only one or the other is - * set, flush only when the relevant condition is hit. - */ - private boolean shouldFlush() { - long flushDelay = zks.getFlushDelay(); - long maxBatchSize = zks.getMaxBatchSize(); - if ((flushDelay > 0) && (getRemainingDelay() == 0)) { - return true; - } - return (maxBatchSize > 0) && (toFlush.size() >= maxBatchSize); - } - - /** - * used by tests to check for changing - * snapcounts - * @param size - */ - public static void setSnapSizeInBytes(long size) { - snapSizeInBytes = size; - } - - private boolean shouldSnapshot() { - int logCount = zks.getZKDatabase().getTxnCount(); - long logSize = zks.getZKDatabase().getTxnSize(); - return (logCount > (snapCount / 2 + randRoll)) - || (snapSizeInBytes > 0 && logSize > (snapSizeInBytes / 2 + randSize)); - } - - private void resetSnapshotStats() { - randRoll = ThreadLocalRandom.current().nextInt(snapCount / 2); - randSize = Math.abs(ThreadLocalRandom.current().nextLong() % (snapSizeInBytes / 2)); - } - - @Override - public void run() { - try { - // we do this in an attempt to ensure that not all of the servers - // in the ensemble take a snapshot at the same time - resetSnapshotStats(); - lastFlushTime = Time.currentElapsedTime(); - while (true) { - ServerMetrics.getMetrics().SYNC_PROCESSOR_QUEUE_SIZE.add(queuedRequests.size()); - - long pollTime = Math.min(zks.getMaxWriteQueuePollTime(), getRemainingDelay()); - Request si = queuedRequests.poll(pollTime, TimeUnit.MILLISECONDS); - if (si == null) { - /* We timed out looking for more writes to batch, go ahead and flush immediately */ - flush(); - si = queuedRequests.take(); - } - - if (si == REQUEST_OF_DEATH) { - break; - } - - if (si == TURN_FORWARDING_DELAY_ON_REQUEST) { - nextProcessor.startDelaying(); - continue; - } - if (si == TURN_FORWARDING_DELAY_OFF_REQUEST) { - nextProcessor.flushAndStopDelaying(); - continue; - } - - if (si instanceof FlushRequest) { - flush(); - ((FlushRequest) si).latch.countDown(); - continue; - } - - long startProcessTime = Time.currentElapsedTime(); - ServerMetrics.getMetrics().SYNC_PROCESSOR_QUEUE_TIME.add(startProcessTime - si.syncQueueStartTime); - - // track the number of records written to the log - if (!si.isThrottled() && zks.getZKDatabase().append(si)) { - if (shouldSnapshot()) { - resetSnapshotStats(); - // roll the log - zks.getZKDatabase().rollLog(); - // take a snapshot - if (!snapThreadMutex.tryAcquire()) { - LOG.warn("Too busy to snap, skipping"); - } else { - new ZooKeeperThread("Snapshot Thread") { - public void run() { - try { - zks.takeSnapshot(); - } catch (Exception e) { - LOG.warn("Unexpected exception", e); - } finally { - snapThreadMutex.release(); - } - } - }.start(); - } - } - } else if (toFlush.isEmpty()) { - // optimization for read heavy workloads - // iff this is a read or a throttled request(which doesn't need to be written to the disk), - // and there are no pending flushes (writes), then just pass this to the next processor - if (nextProcessor != null) { - nextProcessor.processRequest(si); - nextProcessor.flush(); - } - continue; - } - toFlush.add(si); - if (shouldFlush()) { - flush(); - } - ServerMetrics.getMetrics().SYNC_PROCESS_TIME.add(Time.currentElapsedTime() - startProcessTime); - } - } catch (Throwable t) { - handleException(this.getName(), t); - } - LOG.info("SyncRequestProcessor exited!"); - } - - /** Flushes all pending writes, and waits for this to complete. */ - public void syncFlush() throws InterruptedException { - FlushRequest marker = new FlushRequest(); - queuedRequests.add(marker); - marker.latch.await(); - } - - public void setDelayForwarding(boolean delayForwarding) { - queuedRequests.add(delayForwarding ? TURN_FORWARDING_DELAY_ON_REQUEST : TURN_FORWARDING_DELAY_OFF_REQUEST); - } - - private void flush() throws IOException, RequestProcessorException { - if (this.toFlush.isEmpty()) { - return; - } - - ServerMetrics.getMetrics().BATCH_SIZE.add(toFlush.size()); - - long flushStartTime = Time.currentElapsedTime(); - zks.getZKDatabase().commit(); - ServerMetrics.getMetrics().SYNC_PROCESSOR_FLUSH_TIME.add(Time.currentElapsedTime() - flushStartTime); - - if (this.nextProcessor == null) { - this.toFlush.clear(); - } else { - while (!this.toFlush.isEmpty()) { - final Request i = this.toFlush.remove(); - long latency = Time.currentElapsedTime() - i.syncQueueStartTime; - ServerMetrics.getMetrics().SYNC_PROCESSOR_QUEUE_AND_FLUSH_TIME.add(latency); - this.nextProcessor.processRequest(i); - } - nextProcessor.flush(); - } - lastFlushTime = Time.currentElapsedTime(); - } - - public void shutdown() { - LOG.info("Shutting down"); - queuedRequests.add(REQUEST_OF_DEATH); - try { - this.join(); - this.flush(); - } catch (InterruptedException e) { - LOG.warn("Interrupted while wating for {} to finish", this); - Thread.currentThread().interrupt(); - } catch (IOException e) { - LOG.warn("Got IO exception during shutdown"); - } catch (RequestProcessorException e) { - LOG.warn("Got request processor exception during shutdown"); - } - if (nextProcessor != null) { - nextProcessor.shutdown(); - } - } - - public void processRequest(final Request request) { - Objects.requireNonNull(request, "Request cannot be null"); - - request.syncQueueStartTime = Time.currentElapsedTime(); - queuedRequests.add(request); - ServerMetrics.getMetrics().SYNC_PROCESSOR_QUEUED.add(1); - } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java deleted file mode 100644 index 114d2987fe2..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.apache.zookeeper.server; - -import com.yahoo.vespa.zookeeper.Configurator; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.logging.Logger; - -/** - * Overrides secure setting with value from {@link Configurator}. - * Workaround for incorrect handling of clientSecurePort in combination with ZooKeeper Dynamic Reconfiguration in 3.6.2 - * See https://issues.apache.org/jira/browse/ZOOKEEPER-3577. - * - * Using package {@link org.apache.zookeeper.server} as {@link NettyServerCnxnFactory#NettyServerCnxnFactory()} is package-private. - * - * @author bjorncs - */ -public class VespaNettyServerCnxnFactory extends NettyServerCnxnFactory { - - private static final Logger log = Logger.getLogger(VespaNettyServerCnxnFactory.class.getName()); - - private final boolean isSecure; - - public VespaNettyServerCnxnFactory() { - super(); - this.isSecure = Configurator.VespaNettyServerCnxnFactory_isSecure; - boolean portUnificationEnabled = Boolean.getBoolean(NettyServerCnxnFactory.PORT_UNIFICATION_KEY); - log.info(String.format("For %h: isSecure=%b, portUnification=%b", this, isSecure, portUnificationEnabled)); - } - - @Override - public void configure(InetSocketAddress addr, int maxClientCnxns, int backlog, boolean secure) throws IOException { - log.info(String.format("For %h: configured() invoked with parameter 'secure'=%b, overridden to %b", this, secure, isSecure)); - super.configure(addr, maxClientCnxns, backlog, isSecure); - } -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java deleted file mode 100644 index 00af31b46d4..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java +++ /dev/null @@ -1,2412 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server; - -import java.io.BufferedInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintWriter; -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.BiConsumer; -import java.util.zip.Adler32; -import java.util.zip.CheckedInputStream; -import javax.security.sasl.SaslException; -import org.apache.jute.BinaryInputArchive; -import org.apache.jute.BinaryOutputArchive; -import org.apache.jute.InputArchive; -import org.apache.jute.Record; -import org.apache.zookeeper.Environment; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.KeeperException.Code; -import org.apache.zookeeper.KeeperException.SessionExpiredException; -import org.apache.zookeeper.Quotas; -import org.apache.zookeeper.StatsTrack; -import org.apache.zookeeper.Version; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.ZooDefs.OpCode; -import org.apache.zookeeper.ZookeeperBanner; -import org.apache.zookeeper.common.PathUtils; -import org.apache.zookeeper.common.StringUtils; -import org.apache.zookeeper.common.Time; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.data.StatPersisted; -import org.apache.zookeeper.jmx.MBeanRegistry; -import org.apache.zookeeper.metrics.MetricsContext; -import org.apache.zookeeper.proto.AuthPacket; -import org.apache.zookeeper.proto.ConnectRequest; -import org.apache.zookeeper.proto.ConnectResponse; -import org.apache.zookeeper.proto.CreateRequest; -import org.apache.zookeeper.proto.DeleteRequest; -import org.apache.zookeeper.proto.GetSASLRequest; -import org.apache.zookeeper.proto.ReplyHeader; -import org.apache.zookeeper.proto.RequestHeader; -import org.apache.zookeeper.proto.SetACLRequest; -import org.apache.zookeeper.proto.SetDataRequest; -import org.apache.zookeeper.proto.SetSASLResponse; -import org.apache.zookeeper.server.DataTree.ProcessTxnResult; -import org.apache.zookeeper.server.RequestProcessor.RequestProcessorException; -import org.apache.zookeeper.server.ServerCnxn.CloseRequestException; -import org.apache.zookeeper.server.SessionTracker.Session; -import org.apache.zookeeper.server.SessionTracker.SessionExpirer; -import org.apache.zookeeper.server.auth.ProviderRegistry; -import org.apache.zookeeper.server.auth.ServerAuthenticationProvider; -import org.apache.zookeeper.server.persistence.FileTxnSnapLog; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.apache.zookeeper.server.quorum.ReadOnlyZooKeeperServer; -import org.apache.zookeeper.server.util.JvmPauseMonitor; -import org.apache.zookeeper.server.util.OSMXBean; -import org.apache.zookeeper.server.util.QuotaMetricsUtils; -import org.apache.zookeeper.server.util.RequestPathMetricsCollector; -import org.apache.zookeeper.txn.CreateSessionTxn; -import org.apache.zookeeper.txn.TxnDigest; -import org.apache.zookeeper.txn.TxnHeader; -import org.apache.zookeeper.util.ServiceUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class implements a simple standalone ZooKeeperServer. It sets up the - * following chain of RequestProcessors to process requests: - * PrepRequestProcessor -> SyncRequestProcessor -> FinalRequestProcessor - */ -public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { - - protected static final Logger LOG; - private static final RateLogger RATE_LOGGER; - - public static final String GLOBAL_OUTSTANDING_LIMIT = "zookeeper.globalOutstandingLimit"; - - public static final String ENABLE_EAGER_ACL_CHECK = "zookeeper.enableEagerACLCheck"; - public static final String SKIP_ACL = "zookeeper.skipACL"; - public static final String ENFORCE_QUOTA = "zookeeper.enforceQuota"; - - // When enabled, will check ACL constraints appertained to the requests first, - // before sending the requests to the quorum. - static boolean enableEagerACLCheck; - - static final boolean skipACL; - - public static final boolean enforceQuota; - - public static final String SASL_SUPER_USER = "zookeeper.superUser"; - - public static final String ALLOW_SASL_FAILED_CLIENTS = "zookeeper.allowSaslFailedClients"; - public static final String ZOOKEEPER_DIGEST_ENABLED = "zookeeper.digest.enabled"; - private static boolean digestEnabled; - - public static final String ZOOKEEPER_SERIALIZE_LAST_PROCESSED_ZXID_ENABLED = "zookeeper.serializeLastProcessedZxid.enabled"; - private static boolean serializeLastProcessedZxidEnabled; - - // Add a enable/disable option for now, we should remove this one when - // this feature is confirmed to be stable - public static final String CLOSE_SESSION_TXN_ENABLED = "zookeeper.closeSessionTxn.enabled"; - private static boolean closeSessionTxnEnabled = true; - private volatile CountDownLatch restoreLatch; - - static { - LOG = LoggerFactory.getLogger(ZooKeeperServer.class); - - RATE_LOGGER = new RateLogger(LOG); - - ZookeeperBanner.printBanner(LOG); - - Environment.logEnv("Server environment:", LOG); - - enableEagerACLCheck = Boolean.getBoolean(ENABLE_EAGER_ACL_CHECK); - LOG.info("{} = {}", ENABLE_EAGER_ACL_CHECK, enableEagerACLCheck); - - skipACL = System.getProperty(SKIP_ACL, "no").equals("yes"); - if (skipACL) { - LOG.info("{}==\"yes\", ACL checks will be skipped", SKIP_ACL); - } - - enforceQuota = Boolean.parseBoolean(System.getProperty(ENFORCE_QUOTA, "false")); - if (enforceQuota) { - LOG.info("{} = {}, Quota Enforce enables", ENFORCE_QUOTA, enforceQuota); - } - - digestEnabled = Boolean.parseBoolean(System.getProperty(ZOOKEEPER_DIGEST_ENABLED, "true")); - LOG.info("{} = {}", ZOOKEEPER_DIGEST_ENABLED, digestEnabled); - - closeSessionTxnEnabled = Boolean.parseBoolean( - System.getProperty(CLOSE_SESSION_TXN_ENABLED, "true")); - LOG.info("{} = {}", CLOSE_SESSION_TXN_ENABLED, closeSessionTxnEnabled); - - setSerializeLastProcessedZxidEnabled(Boolean.parseBoolean( - System.getProperty(ZOOKEEPER_SERIALIZE_LAST_PROCESSED_ZXID_ENABLED, "true"))); - } - - // @VisibleForTesting - public static boolean isEnableEagerACLCheck() { - return enableEagerACLCheck; - } - - // @VisibleForTesting - public static void setEnableEagerACLCheck(boolean enabled) { - ZooKeeperServer.enableEagerACLCheck = enabled; - LOG.info("Update {} to {}", ENABLE_EAGER_ACL_CHECK, enabled); - } - - public static boolean isCloseSessionTxnEnabled() { - return closeSessionTxnEnabled; - } - - public static void setCloseSessionTxnEnabled(boolean enabled) { - ZooKeeperServer.closeSessionTxnEnabled = enabled; - LOG.info("Update {} to {}", CLOSE_SESSION_TXN_ENABLED, - ZooKeeperServer.closeSessionTxnEnabled); - } - - protected ZooKeeperServerBean jmxServerBean; - protected DataTreeBean jmxDataTreeBean; - - public static final int DEFAULT_TICK_TIME = 3000; - protected int tickTime = DEFAULT_TICK_TIME; - public static final int DEFAULT_THROTTLED_OP_WAIT_TIME = 0; // disabled - protected static volatile int throttledOpWaitTime = - Integer.getInteger("zookeeper.throttled_op_wait_time", DEFAULT_THROTTLED_OP_WAIT_TIME); - /** value of -1 indicates unset, use default */ - protected int minSessionTimeout = -1; - /** value of -1 indicates unset, use default */ - protected int maxSessionTimeout = -1; - /** Socket listen backlog. Value of -1 indicates unset */ - protected int listenBacklog = -1; - protected SessionTracker sessionTracker; - private FileTxnSnapLog txnLogFactory = null; - private ZKDatabase zkDb; - private ResponseCache readResponseCache; - private ResponseCache getChildrenResponseCache; - private final AtomicLong hzxid = new AtomicLong(0); - public static final Exception ok = new Exception("No prob"); - protected RequestProcessor firstProcessor; - protected JvmPauseMonitor jvmPauseMonitor; - protected volatile State state = State.INITIAL; - private boolean isResponseCachingEnabled = true; - /* contains the configuration file content read at startup */ - protected String initialConfig; - protected boolean reconfigEnabled; - private final RequestPathMetricsCollector requestPathMetricsCollector; - private static final int DEFAULT_SNAP_COUNT = 100000; - private static final int DEFAULT_GLOBAL_OUTSTANDING_LIMIT = 1000; - - private boolean localSessionEnabled = false; - protected enum State { - INITIAL, - RUNNING, - SHUTDOWN, - ERROR - } - - /** - * This is the secret that we use to generate passwords. For the moment, - * it's more of a checksum that's used in reconnection, which carries no - * security weight, and is treated internally as if it carries no - * security weight. - */ - private static final long superSecret = 0XB3415C00L; - - private final AtomicInteger requestsInProcess = new AtomicInteger(0); - final Deque<ChangeRecord> outstandingChanges = new ArrayDeque<>(); - // this data structure must be accessed under the outstandingChanges lock - final Map<String, ChangeRecord> outstandingChangesForPath = new HashMap<>(); - - protected ServerCnxnFactory serverCnxnFactory; - protected ServerCnxnFactory secureServerCnxnFactory; - - private final ServerStats serverStats; - private final ZooKeeperServerListener listener; - private ZooKeeperServerShutdownHandler zkShutdownHandler; - private volatile int createSessionTrackerServerId = 1; - - private static final String FLUSH_DELAY = "zookeeper.flushDelay"; - private static volatile long flushDelay; - private static final String MAX_WRITE_QUEUE_POLL_SIZE = "zookeeper.maxWriteQueuePollTime"; - private static volatile long maxWriteQueuePollTime; - private static final String MAX_BATCH_SIZE = "zookeeper.maxBatchSize"; - private static volatile int maxBatchSize; - - /** - * Starting size of read and write ByteArroyOuputBuffers. Default is 32 bytes. - * Flag not used for small transfers like connectResponses. - */ - public static final String INT_BUFFER_STARTING_SIZE_BYTES = "zookeeper.intBufferStartingSizeBytes"; - public static final int DEFAULT_STARTING_BUFFER_SIZE = 1024; - public static final int intBufferStartingSizeBytes; - - public static final String GET_DATA_RESPONSE_CACHE_SIZE = "zookeeper.maxResponseCacheSize"; - public static final String GET_CHILDREN_RESPONSE_CACHE_SIZE = "zookeeper.maxGetChildrenResponseCacheSize"; - - static { - long configuredFlushDelay = Long.getLong(FLUSH_DELAY, 0); - setFlushDelay(configuredFlushDelay); - setMaxWriteQueuePollTime(Long.getLong(MAX_WRITE_QUEUE_POLL_SIZE, configuredFlushDelay / 3)); - setMaxBatchSize(Integer.getInteger(MAX_BATCH_SIZE, 1000)); - - intBufferStartingSizeBytes = Integer.getInteger(INT_BUFFER_STARTING_SIZE_BYTES, DEFAULT_STARTING_BUFFER_SIZE); - - if (intBufferStartingSizeBytes < 32) { - String msg = "Buffer starting size (" + intBufferStartingSizeBytes + ") must be greater than or equal to 32. " - + "Configure with \"-Dzookeeper.intBufferStartingSizeBytes=<size>\" "; - LOG.error(msg); - throw new IllegalArgumentException(msg); - } - - LOG.info("{} = {}", INT_BUFFER_STARTING_SIZE_BYTES, intBufferStartingSizeBytes); - } - - // Connection throttling - private final BlueThrottle connThrottle = new BlueThrottle(); - - private RequestThrottler requestThrottler; - public static final String SNAP_COUNT = "zookeeper.snapCount"; - - /** - * This setting sets a limit on the total number of large requests that - * can be inflight and is designed to prevent ZooKeeper from accepting - * too many large requests such that the JVM runs out of usable heap and - * ultimately crashes. - * - * The limit is enforced by the {@link #checkRequestSizeWhenReceivingMessage(int)} - * method which is called by the connection layer ({@link NIOServerCnxn}, - * {@link NettyServerCnxn}) before allocating a byte buffer and pulling - * data off the TCP socket. The limit is then checked again by the - * ZooKeeper server in {@link #processPacket(ServerCnxn, RequestHeader, RequestRecord)} which - * also atomically updates {@link #currentLargeRequestBytes}. The request is - * then marked as a large request, with the request size stored in the Request - * object so that it can later be decremented from {@link #currentLargeRequestBytes}. - * - * When a request is completed or dropped, the relevant code path calls the - * {@link #requestFinished(Request)} method which performs the decrement if - * needed. - */ - private volatile int largeRequestMaxBytes = 100 * 1024 * 1024; - - /** - * The size threshold after which a request is considered a large request - * and is checked against the large request byte limit. - */ - private volatile int largeRequestThreshold = -1; - - private final AtomicInteger currentLargeRequestBytes = new AtomicInteger(0); - - private final AuthenticationHelper authHelper = new AuthenticationHelper(); - - void removeCnxn(ServerCnxn cnxn) { - zkDb.removeCnxn(cnxn); - } - - /** - * Creates a ZooKeeperServer instance. Nothing is setup, use the setX - * methods to prepare the instance (eg datadir, datalogdir, ticktime, - * builder, etc...) - * - */ - public ZooKeeperServer() { - listener = new ZooKeeperServerListenerImpl(this); - serverStats = new ServerStats(this); - this.requestPathMetricsCollector = new RequestPathMetricsCollector(); - } - - /** - * Keeping this constructor for backward compatibility - */ - public ZooKeeperServer(FileTxnSnapLog txnLogFactory, int tickTime, int minSessionTimeout, int maxSessionTimeout, int clientPortListenBacklog, ZKDatabase zkDb, String initialConfig) { - this(txnLogFactory, tickTime, minSessionTimeout, maxSessionTimeout, clientPortListenBacklog, zkDb, initialConfig, QuorumPeerConfig.isReconfigEnabled()); - } - - /** - * * Creates a ZooKeeperServer instance. It sets everything up, but doesn't - * actually start listening for clients until run() is invoked. - * - */ - public ZooKeeperServer(FileTxnSnapLog txnLogFactory, int tickTime, int minSessionTimeout, int maxSessionTimeout, int clientPortListenBacklog, ZKDatabase zkDb, String initialConfig, boolean reconfigEnabled) { - serverStats = new ServerStats(this); - this.txnLogFactory = txnLogFactory; - this.txnLogFactory.setServerStats(this.serverStats); - this.zkDb = zkDb; - this.tickTime = tickTime; - setMinSessionTimeout(minSessionTimeout); - setMaxSessionTimeout(maxSessionTimeout); - this.listenBacklog = clientPortListenBacklog; - this.reconfigEnabled = reconfigEnabled; - - listener = new ZooKeeperServerListenerImpl(this); - - readResponseCache = new ResponseCache(Integer.getInteger( - GET_DATA_RESPONSE_CACHE_SIZE, - ResponseCache.DEFAULT_RESPONSE_CACHE_SIZE), "getData"); - - getChildrenResponseCache = new ResponseCache(Integer.getInteger( - GET_CHILDREN_RESPONSE_CACHE_SIZE, - ResponseCache.DEFAULT_RESPONSE_CACHE_SIZE), "getChildren"); - - this.initialConfig = initialConfig; - - this.requestPathMetricsCollector = new RequestPathMetricsCollector(); - - this.initLargeRequestThrottlingSettings(); - - LOG.info( - "Created server with" - + " tickTime {} ms" - + " minSessionTimeout {} ms" - + " maxSessionTimeout {} ms" - + " clientPortListenBacklog {}" - + " dataLogdir {}" - + " snapdir {}", - tickTime, - getMinSessionTimeout(), - getMaxSessionTimeout(), - getClientPortListenBacklog(), - txnLogFactory.getDataLogDir(), - txnLogFactory.getSnapDir()); - } - - public String getInitialConfig() { - return initialConfig; - } - - /** - * Adds JvmPauseMonitor and calls - * {@link #ZooKeeperServer(FileTxnSnapLog, int, int, int, int, ZKDatabase, String)} - * - */ - public ZooKeeperServer(JvmPauseMonitor jvmPauseMonitor, FileTxnSnapLog txnLogFactory, int tickTime, int minSessionTimeout, int maxSessionTimeout, int clientPortListenBacklog, ZKDatabase zkDb, String initialConfig) { - this(txnLogFactory, tickTime, minSessionTimeout, maxSessionTimeout, clientPortListenBacklog, zkDb, initialConfig, QuorumPeerConfig.isReconfigEnabled()); - this.jvmPauseMonitor = jvmPauseMonitor; - if (jvmPauseMonitor != null) { - LOG.info("Added JvmPauseMonitor to server"); - } - } - - /** - * creates a zookeeperserver instance. - * @param txnLogFactory the file transaction snapshot logging class - * @param tickTime the ticktime for the server - */ - public ZooKeeperServer(FileTxnSnapLog txnLogFactory, int tickTime, String initialConfig) { - this(txnLogFactory, tickTime, -1, -1, -1, new ZKDatabase(txnLogFactory), initialConfig, QuorumPeerConfig.isReconfigEnabled()); - } - - public ServerStats serverStats() { - return serverStats; - } - - public RequestPathMetricsCollector getRequestPathMetricsCollector() { - return requestPathMetricsCollector; - } - - public BlueThrottle connThrottle() { - return connThrottle; - } - - public void dumpConf(PrintWriter pwriter) { - pwriter.print("clientPort="); - pwriter.println(getClientPort()); - pwriter.print("secureClientPort="); - pwriter.println(getSecureClientPort()); - pwriter.print("dataDir="); - pwriter.println(zkDb.snapLog.getSnapDir().getAbsolutePath()); - pwriter.print("dataDirSize="); - pwriter.println(getDataDirSize()); - pwriter.print("dataLogDir="); - pwriter.println(zkDb.snapLog.getDataLogDir().getAbsolutePath()); - pwriter.print("dataLogSize="); - pwriter.println(getLogDirSize()); - pwriter.print("tickTime="); - pwriter.println(getTickTime()); - pwriter.print("maxClientCnxns="); - pwriter.println(getMaxClientCnxnsPerHost()); - pwriter.print("minSessionTimeout="); - pwriter.println(getMinSessionTimeout()); - pwriter.print("maxSessionTimeout="); - pwriter.println(getMaxSessionTimeout()); - pwriter.print("clientPortListenBacklog="); - pwriter.println(getClientPortListenBacklog()); - - pwriter.print("serverId="); - pwriter.println(getServerId()); - } - - public ZooKeeperServerConf getConf() { - return new ZooKeeperServerConf( - getClientPort(), - zkDb.snapLog.getSnapDir().getAbsolutePath(), - zkDb.snapLog.getDataLogDir().getAbsolutePath(), - getTickTime(), - getMaxClientCnxnsPerHost(), - getMinSessionTimeout(), - getMaxSessionTimeout(), - getServerId(), - getClientPortListenBacklog()); - } - - /** - * This constructor is for backward compatibility with the existing unit - * test code. - * It defaults to FileLogProvider persistence provider. - */ - public ZooKeeperServer(File snapDir, File logDir, int tickTime) throws IOException { - this(new FileTxnSnapLog(snapDir, logDir), tickTime, ""); - } - - /** - * Default constructor, relies on the config for its argument values - * - * @throws IOException - */ - public ZooKeeperServer(FileTxnSnapLog txnLogFactory) throws IOException { - this(txnLogFactory, DEFAULT_TICK_TIME, -1, -1, -1, new ZKDatabase(txnLogFactory), "", QuorumPeerConfig.isReconfigEnabled()); - } - - /** - * get the zookeeper database for this server - * @return the zookeeper database for this server - */ - public ZKDatabase getZKDatabase() { - return this.zkDb; - } - - /** - * set the zkdatabase for this zookeeper server - * @param zkDb - */ - public void setZKDatabase(ZKDatabase zkDb) { - this.zkDb = zkDb; - } - - /** - * Restore sessions and data - */ - public void loadData() throws IOException, InterruptedException { - /* - * When a new leader starts executing Leader#lead, it - * invokes this method. The database, however, has been - * initialized before running leader election so that - * the server could pick its zxid for its initial vote. - * It does it by invoking QuorumPeer#getLastLoggedZxid. - * Consequently, we don't need to initialize it once more - * and avoid the penalty of loading it a second time. Not - * reloading it is particularly important for applications - * that host a large database. - * - * The following if block checks whether the database has - * been initialized or not. Note that this method is - * invoked by at least one other method: - * ZooKeeperServer#startdata. - * - * See ZOOKEEPER-1642 for more detail. - */ - if (zkDb.isInitialized()) { - setZxid(zkDb.getDataTreeLastProcessedZxid()); - } else { - setZxid(zkDb.loadDataBase()); - } - - // Clean up dead sessions - zkDb.getSessions().stream() - .filter(session -> zkDb.getSessionWithTimeOuts().get(session) == null) - .forEach(session -> killSession(session, zkDb.getDataTreeLastProcessedZxid())); - - // Make a clean snapshot - takeSnapshot(); - } - - public File takeSnapshot() throws IOException { - return takeSnapshot(false); - } - - public File takeSnapshot(boolean syncSnap) throws IOException { - return takeSnapshot(syncSnap, true, false); - } - - /** - * Takes a snapshot on the server. - * - * @param syncSnap syncSnap sync the snapshot immediately after write - * @param isSevere if true system exist, otherwise throw IOException - * @param fastForwardFromEdits whether fast forward database to the latest recorded transactions - * - * @return file snapshot file object - * @throws IOException - */ - public synchronized File takeSnapshot(boolean syncSnap, boolean isSevere, boolean fastForwardFromEdits) throws IOException { - long start = Time.currentElapsedTime(); - File snapFile = null; - try { - if (fastForwardFromEdits) { - zkDb.fastForwardDataBase(); - } - snapFile = txnLogFactory.save(zkDb.getDataTree(), zkDb.getSessionWithTimeOuts(), syncSnap); - } catch (IOException e) { - if (isSevere) { - LOG.error("Severe unrecoverable error, exiting", e); - // This is a severe error that we cannot recover from, - // so we need to exit - ServiceUtils.requestSystemExit(ExitCode.TXNLOG_ERROR_TAKING_SNAPSHOT.getValue()); - } else { - throw e; - } - } - long elapsed = Time.currentElapsedTime() - start; - LOG.info("Snapshot taken in {} ms", elapsed); - ServerMetrics.getMetrics().SNAPSHOT_TIME.add(elapsed); - return snapFile; - } - - /** - * Restores database from a snapshot. It is used by the restore admin server command. - * - * @param inputStream input stream of snapshot - * @return last processed zxid - */ - public synchronized long restoreFromSnapshot(final InputStream inputStream) throws IOException { - if (inputStream == null) { - throw new IllegalArgumentException("InputStream can not be null when restoring from snapshot"); - } - - long start = Time.currentElapsedTime(); - LOG.info("Before restore database. lastProcessedZxid={}, nodeCount={}，sessionCount={}", - getZKDatabase().getDataTreeLastProcessedZxid(), - getZKDatabase().dataTree.getNodeCount(), - getZKDatabase().getSessionCount()); - - // restore to a new zkDatabase - final ZKDatabase newZKDatabase = new ZKDatabase(this.txnLogFactory); - final CheckedInputStream cis = new CheckedInputStream(new BufferedInputStream(inputStream), new Adler32()); - final InputArchive ia = BinaryInputArchive.getArchive(cis); - newZKDatabase.deserializeSnapshot(ia, cis); - LOG.info("Restored to a new database. lastProcessedZxid={}, nodeCount={}, sessionCount={}", - newZKDatabase.getDataTreeLastProcessedZxid(), - newZKDatabase.dataTree.getNodeCount(), - newZKDatabase.getSessionCount()); - - // create a CountDownLatch - restoreLatch = new CountDownLatch(1); - - try { - // set to the new zkDatabase - setZKDatabase(newZKDatabase); - - // re-create SessionTrack - createSessionTracker(); - } finally { - // unblock request submission - restoreLatch.countDown(); - restoreLatch = null; - } - - LOG.info("After restore database. lastProcessedZxid={}, nodeCount={}, sessionCount={}", - getZKDatabase().getDataTreeLastProcessedZxid(), - getZKDatabase().dataTree.getNodeCount(), - getZKDatabase().getSessionCount()); - - long elapsed = Time.currentElapsedTime() - start; - LOG.info("Restore taken in {} ms", elapsed); - ServerMetrics.getMetrics().RESTORE_TIME.add(elapsed); - - return getLastProcessedZxid(); - } - - public boolean shouldForceWriteInitialSnapshotAfterLeaderElection() { - return txnLogFactory.shouldForceWriteInitialSnapshotAfterLeaderElection(); - } - - @Override - public long getDataDirSize() { - if (zkDb == null) { - return 0L; - } - File path = zkDb.snapLog.getSnapDir(); - return getDirSize(path); - } - - @Override - public long getLogDirSize() { - if (zkDb == null) { - return 0L; - } - File path = zkDb.snapLog.getDataLogDir(); - return getDirSize(path); - } - - private long getDirSize(File file) { - long size = 0L; - if (file.isDirectory()) { - File[] files = file.listFiles(); - if (files != null) { - for (File f : files) { - size += getDirSize(f); - } - } - } else { - size = file.length(); - } - return size; - } - - public long getZxid() { - return hzxid.get(); - } - - public SessionTracker getSessionTracker() { - return sessionTracker; - } - - long getNextZxid() { - return hzxid.incrementAndGet(); - } - - public void setZxid(long zxid) { - hzxid.set(zxid); - } - - private void close(long sessionId) { - Request si = new Request(null, sessionId, 0, OpCode.closeSession, null, null); - submitRequest(si); - } - - public void closeSession(long sessionId) { - LOG.info("Closing session 0x{}", Long.toHexString(sessionId)); - - // we do not want to wait for a session close. send it as soon as we - // detect it! - close(sessionId); - } - - protected void killSession(long sessionId, long zxid) { - zkDb.killSession(sessionId, zxid); - if (LOG.isTraceEnabled()) { - ZooTrace.logTraceMessage( - LOG, - ZooTrace.SESSION_TRACE_MASK, - "ZooKeeperServer --- killSession: 0x" + Long.toHexString(sessionId)); - } - if (sessionTracker != null) { - sessionTracker.removeSession(sessionId); - } - } - - public void expire(Session session) { - long sessionId = session.getSessionId(); - LOG.info( - "Expiring session 0x{}, timeout of {}ms exceeded", - Long.toHexString(sessionId), - session.getTimeout()); - close(sessionId); - } - - public void expire(long sessionId) { - LOG.info("forcibly expiring session 0x{}", Long.toHexString(sessionId)); - - close(sessionId); - } - - public static class MissingSessionException extends IOException { - - private static final long serialVersionUID = 7467414635467261007L; - - public MissingSessionException(String msg) { - super(msg); - } - - } - - void touch(ServerCnxn cnxn) throws MissingSessionException { - if (cnxn == null) { - return; - } - long id = cnxn.getSessionId(); - int to = cnxn.getSessionTimeout(); - if (!sessionTracker.touchSession(id, to)) { - throw new MissingSessionException("No session with sessionid 0x" - + Long.toHexString(id) - + " exists, probably expired and removed"); - } - } - - protected void registerJMX() { - // register with JMX - try { - jmxServerBean = new ZooKeeperServerBean(this); - MBeanRegistry.getInstance().register(jmxServerBean, null); - - try { - jmxDataTreeBean = new DataTreeBean(zkDb.getDataTree()); - MBeanRegistry.getInstance().register(jmxDataTreeBean, jmxServerBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - jmxDataTreeBean = null; - } - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - jmxServerBean = null; - } - } - - public void startdata() throws IOException, InterruptedException { - //check to see if zkDb is not null - if (zkDb == null) { - zkDb = new ZKDatabase(this.txnLogFactory); - } - if (!zkDb.isInitialized()) { - loadData(); - } - } - - public synchronized void startup() { - startupWithServerState(State.RUNNING); - } - - public synchronized void startupWithoutServing() { - startupWithServerState(State.INITIAL); - } - - public synchronized void startServing() { - setState(State.RUNNING); - notifyAll(); - } - - private void startupWithServerState(State state) { - if (sessionTracker == null) { - createSessionTracker(); - } - startSessionTracker(); - setupRequestProcessors(); - - startRequestThrottler(); - - registerJMX(); - - startJvmPauseMonitor(); - - registerMetrics(); - - setState(state); - - requestPathMetricsCollector.start(); - - localSessionEnabled = sessionTracker.isLocalSessionsEnabled(); - - notifyAll(); - } - - protected void startJvmPauseMonitor() { - if (this.jvmPauseMonitor != null) { - this.jvmPauseMonitor.serviceStart(); - } - } - - protected void startRequestThrottler() { - requestThrottler = createRequestThrottler(); - requestThrottler.start(); - } - - protected RequestThrottler createRequestThrottler() { - return new RequestThrottler(this); - } - - protected void setupRequestProcessors() { - RequestProcessor finalProcessor = new FinalRequestProcessor(this); - RequestProcessor syncProcessor = new SyncRequestProcessor(this, finalProcessor); - ((SyncRequestProcessor) syncProcessor).start(); - firstProcessor = new PrepRequestProcessor(this, syncProcessor); - ((PrepRequestProcessor) firstProcessor).start(); - } - - public ZooKeeperServerListener getZooKeeperServerListener() { - return listener; - } - - /** - * Change the server ID used by {@link #createSessionTracker()}. Must be called prior to - * {@link #startup()} being called - * - * @param newId ID to use - */ - public void setCreateSessionTrackerServerId(int newId) { - createSessionTrackerServerId = newId; - } - - protected void createSessionTracker() { - sessionTracker = new SessionTrackerImpl(this, zkDb.getSessionWithTimeOuts(), tickTime, createSessionTrackerServerId, getZooKeeperServerListener()); - } - - protected void startSessionTracker() { - ((SessionTrackerImpl) sessionTracker).start(); - } - - /** - * Sets the state of ZooKeeper server. After changing the state, it notifies - * the server state change to a registered shutdown handler, if any. - * <p> - * The following are the server state transitions: - * <ul><li>During startup the server will be in the INITIAL state.</li> - * <li>After successfully starting, the server sets the state to RUNNING. - * </li> - * <li>The server transitions to the ERROR state if it hits an internal - * error. {@link ZooKeeperServerListenerImpl} notifies any critical resource - * error events, e.g., SyncRequestProcessor not being able to write a txn to - * disk.</li> - * <li>During shutdown the server sets the state to SHUTDOWN, which - * corresponds to the server not running.</li> - * - * <li>During maintenance (e.g. restore) the server sets the state to MAINTENANCE - * </li></ul> - * - * @param state new server state. - */ - protected void setState(State state) { - this.state = state; - // Notify server state changes to the registered shutdown handler, if any. - if (zkShutdownHandler != null) { - zkShutdownHandler.handle(state); - } else { - LOG.debug( - "ZKShutdownHandler is not registered, so ZooKeeper server" - + " won't take any action on ERROR or SHUTDOWN server state changes"); - } - } - - /** - * This can be used while shutting down the server to see whether the server - * is already shutdown or not. - * - * @return true if the server is running or server hits an error, false - * otherwise. - */ - protected boolean canShutdown() { - return state == State.RUNNING || state == State.ERROR; - } - - /** - * @return true if the server is running, false otherwise. - */ - public boolean isRunning() { - return state == State.RUNNING; - } - - public void shutdown() { - shutdown(false); - } - - /** - * Shut down the server instance - * @param fullyShutDown true if another server using the same database will not replace this one in the same process - */ - public synchronized void shutdown(boolean fullyShutDown) { - if (!canShutdown()) { - if (fullyShutDown && zkDb != null) { - zkDb.clear(); - } - LOG.debug("ZooKeeper server is not running, so not proceeding to shutdown!"); - return; - } - LOG.info("shutting down"); - - // new RuntimeException("Calling shutdown").printStackTrace(); - setState(State.SHUTDOWN); - - // unregister all metrics that are keeping a strong reference to this object - // subclasses will do their specific clean up - unregisterMetrics(); - - if (requestThrottler != null) { - requestThrottler.shutdown(); - } - - // Since sessionTracker and syncThreads poll we just have to - // set running to false and they will detect it during the poll - // interval. - if (sessionTracker != null) { - sessionTracker.shutdown(); - } - if (firstProcessor != null) { - firstProcessor.shutdown(); - } - if (jvmPauseMonitor != null) { - jvmPauseMonitor.serviceStop(); - } - - if (zkDb != null) { - if (fullyShutDown) { - zkDb.clear(); - } else { - // else there is no need to clear the database - // * When a new quorum is established we can still apply the diff - // on top of the same zkDb data - // * If we fetch a new snapshot from leader, the zkDb will be - // cleared anyway before loading the snapshot - try { - // This will fast-forward the database to the latest recorded transactions - zkDb.fastForwardDataBase(); - } catch (IOException e) { - LOG.error("Error updating DB", e); - zkDb.clear(); - } - } - } - - requestPathMetricsCollector.shutdown(); - unregisterJMX(); - } - - protected void unregisterJMX() { - // unregister from JMX - try { - if (jmxDataTreeBean != null) { - MBeanRegistry.getInstance().unregister(jmxDataTreeBean); - } - } catch (Exception e) { - LOG.warn("Failed to unregister with JMX", e); - } - try { - if (jmxServerBean != null) { - MBeanRegistry.getInstance().unregister(jmxServerBean); - } - } catch (Exception e) { - LOG.warn("Failed to unregister with JMX", e); - } - jmxServerBean = null; - jmxDataTreeBean = null; - } - - public void incInProcess() { - requestsInProcess.incrementAndGet(); - } - - public void decInProcess() { - requestsInProcess.decrementAndGet(); - if (requestThrottler != null) { - requestThrottler.throttleWake(); - } - } - - public int getInProcess() { - return requestsInProcess.get(); - } - - public int getInflight() { - return requestThrottleInflight(); - } - - private int requestThrottleInflight() { - if (requestThrottler != null) { - return requestThrottler.getInflight(); - } - return 0; - } - - static class PrecalculatedDigest { - final long nodeDigest; - final long treeDigest; - - PrecalculatedDigest(long nodeDigest, long treeDigest) { - this.nodeDigest = nodeDigest; - this.treeDigest = treeDigest; - } - } - - - /** - * This structure is used to facilitate information sharing between PrepRP - * and FinalRP. - */ - static class ChangeRecord { - PrecalculatedDigest precalculatedDigest; - byte[] data; - - ChangeRecord(long zxid, String path, StatPersisted stat, int childCount, List<ACL> acl) { - this.zxid = zxid; - this.path = path; - this.stat = stat; - this.childCount = childCount; - this.acl = acl; - } - - long zxid; - - String path; - - StatPersisted stat; /* Make sure to create a new object when changing */ - - int childCount; - - List<ACL> acl; /* Make sure to create a new object when changing */ - - ChangeRecord duplicate(long zxid) { - StatPersisted stat = new StatPersisted(); - if (this.stat != null) { - DataTree.copyStatPersisted(this.stat, stat); - } - ChangeRecord changeRecord = new ChangeRecord(zxid, path, stat, childCount, - acl == null ? new ArrayList<>() : new ArrayList<>(acl)); - changeRecord.precalculatedDigest = precalculatedDigest; - changeRecord.data = data; - return changeRecord; - } - - } - - byte[] generatePasswd(long id) { - Random r = new Random(id ^ superSecret); - byte[] p = new byte[16]; - r.nextBytes(p); - return p; - } - - protected boolean checkPasswd(long sessionId, byte[] passwd) { - return sessionId != 0 && Arrays.equals(passwd, generatePasswd(sessionId)); - } - - long createSession(ServerCnxn cnxn, byte[] passwd, int timeout) { - if (passwd == null) { - // Possible since it's just deserialized from a packet on the wire. - passwd = new byte[0]; - } - long sessionId = sessionTracker.createSession(timeout); - Random r = new Random(sessionId ^ superSecret); - r.nextBytes(passwd); - CreateSessionTxn txn = new CreateSessionTxn(timeout); - cnxn.setSessionId(sessionId); - Request si = new Request(cnxn, sessionId, 0, OpCode.createSession, RequestRecord.fromRecord(txn), null); - submitRequest(si); - return sessionId; - } - - /** - * set the owner of this session as owner - * @param id the session id - * @param owner the owner of the session - * @throws SessionExpiredException - */ - public void setOwner(long id, Object owner) throws SessionExpiredException { - sessionTracker.setOwner(id, owner); - } - - protected void revalidateSession(ServerCnxn cnxn, long sessionId, int sessionTimeout) throws IOException { - boolean rc = sessionTracker.touchSession(sessionId, sessionTimeout); - if (LOG.isTraceEnabled()) { - ZooTrace.logTraceMessage( - LOG, - ZooTrace.SESSION_TRACE_MASK, - "Session 0x" + Long.toHexString(sessionId) + " is valid: " + rc); - } - finishSessionInit(cnxn, rc); - } - - public void reopenSession(ServerCnxn cnxn, long sessionId, byte[] passwd, int sessionTimeout) throws IOException { - if (checkPasswd(sessionId, passwd)) { - revalidateSession(cnxn, sessionId, sessionTimeout); - } else { - LOG.warn( - "Incorrect password from {} for session 0x{}", - cnxn.getRemoteSocketAddress(), - Long.toHexString(sessionId)); - finishSessionInit(cnxn, false); - } - } - - public void finishSessionInit(ServerCnxn cnxn, boolean valid) { - // register with JMX - try { - if (valid) { - if (serverCnxnFactory != null && serverCnxnFactory.cnxns.contains(cnxn)) { - serverCnxnFactory.registerConnection(cnxn); - } else if (secureServerCnxnFactory != null && secureServerCnxnFactory.cnxns.contains(cnxn)) { - secureServerCnxnFactory.registerConnection(cnxn); - } - } - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - } - - try { - ConnectResponse rsp = new ConnectResponse( - 0, - valid ? cnxn.getSessionTimeout() : 0, - valid ? cnxn.getSessionId() : 0, // send 0 if session is no - // longer valid - valid ? generatePasswd(cnxn.getSessionId()) : new byte[16], - this instanceof ReadOnlyZooKeeperServer); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - BinaryOutputArchive bos = BinaryOutputArchive.getArchive(baos); - bos.writeInt(-1, "len"); - rsp.serialize(bos, "connect"); - baos.close(); - ByteBuffer bb = ByteBuffer.wrap(baos.toByteArray()); - bb.putInt(bb.remaining() - 4).rewind(); - cnxn.sendBuffer(bb); - - if (valid) { - LOG.debug( - "Established session 0x{} with negotiated timeout {} for client {}", - Long.toHexString(cnxn.getSessionId()), - cnxn.getSessionTimeout(), - cnxn.getRemoteSocketAddress()); - cnxn.enableRecv(); - } else { - - LOG.info( - "Invalid session 0x{} for client {}, probably expired", - Long.toHexString(cnxn.getSessionId()), - cnxn.getRemoteSocketAddress()); - cnxn.sendBuffer(ServerCnxnFactory.closeConn); - } - - } catch (Exception e) { - LOG.warn("Exception while establishing session, closing", e); - cnxn.close(ServerCnxn.DisconnectReason.IO_EXCEPTION_IN_SESSION_INIT); - } - } - - public void closeSession(ServerCnxn cnxn, RequestHeader requestHeader) { - closeSession(cnxn.getSessionId()); - } - - public long getServerId() { - return 0; - } - - /** - * If the underlying Zookeeper server support local session, this method - * will set a isLocalSession to true if a request is associated with - * a local session. - * - * @param si - */ - protected void setLocalSessionFlag(Request si) { - } - - public void submitRequest(Request si) { - if (restoreLatch != null) { - try { - LOG.info("Blocking request submission while restore is in progress"); - restoreLatch.await(); - } catch (final InterruptedException e) { - LOG.warn("Unexpected interruption", e); - } - } - enqueueRequest(si); - } - - public void enqueueRequest(Request si) { - if (requestThrottler == null) { - synchronized (this) { - try { - // Since all requests are passed to the request - // processor it should wait for setting up the request - // processor chain. The state will be updated to RUNNING - // after the setup. - while (state == State.INITIAL) { - wait(1000); - } - } catch (InterruptedException e) { - LOG.warn("Unexpected interruption", e); - } - if (requestThrottler == null) { - throw new RuntimeException("Not started"); - } - } - } - requestThrottler.submitRequest(si); - } - - public void submitRequestNow(Request si) { - if (firstProcessor == null) { - synchronized (this) { - try { - // Since all requests are passed to the request - // processor it should wait for setting up the request - // processor chain. The state will be updated to RUNNING - // after the setup. - while (state == State.INITIAL) { - wait(1000); - } - } catch (InterruptedException e) { - LOG.warn("Unexpected interruption", e); - } - if (firstProcessor == null || state != State.RUNNING) { - throw new RuntimeException("Not started"); - } - } - } - try { - touch(si.cnxn); - boolean validpacket = Request.isValid(si.type); - if (validpacket) { - setLocalSessionFlag(si); - firstProcessor.processRequest(si); - if (si.cnxn != null) { - incInProcess(); - } - } else { - LOG.warn("Received packet at server of unknown type {}", si.type); - // Update request accounting/throttling limits - requestFinished(si); - new UnimplementedRequestProcessor().processRequest(si); - } - } catch (MissingSessionException e) { - LOG.debug("Dropping request.", e); - // Update request accounting/throttling limits - requestFinished(si); - } catch (RequestProcessorException e) { - LOG.error("Unable to process request", e); - // Update request accounting/throttling limits - requestFinished(si); - } - } - - public static int getSnapCount() { - int snapCount = Integer.getInteger(SNAP_COUNT, DEFAULT_SNAP_COUNT); - // snapCount must be 2 or more. See org.apache.zookeeper.server.SyncRequestProcessor - if (snapCount < 2) { - LOG.warn("SnapCount should be 2 or more. Now, snapCount is reset to 2"); - snapCount = 2; - } - return snapCount; - } - - public int getGlobalOutstandingLimit() { - return Integer.getInteger(GLOBAL_OUTSTANDING_LIMIT, DEFAULT_GLOBAL_OUTSTANDING_LIMIT); - } - - public static long getSnapSizeInBytes() { - long size = Long.getLong("zookeeper.snapSizeLimitInKb", 4194304L); // 4GB by default - if (size <= 0) { - LOG.info("zookeeper.snapSizeLimitInKb set to a non-positive value {}; disabling feature", size); - } - return size * 1024; // Convert to bytes - } - - public void setServerCnxnFactory(ServerCnxnFactory factory) { - serverCnxnFactory = factory; - } - - public ServerCnxnFactory getServerCnxnFactory() { - return serverCnxnFactory; - } - - public ServerCnxnFactory getSecureServerCnxnFactory() { - return secureServerCnxnFactory; - } - - public void setSecureServerCnxnFactory(ServerCnxnFactory factory) { - secureServerCnxnFactory = factory; - } - - /** - * return the last processed id from the - * datatree - */ - public long getLastProcessedZxid() { - return zkDb.getDataTreeLastProcessedZxid(); - } - - /** - * return the outstanding requests - * in the queue, which haven't been - * processed yet - */ - public long getOutstandingRequests() { - return getInProcess(); - } - - /** - * return the total number of client connections that are alive - * to this server - */ - public int getNumAliveConnections() { - int numAliveConnections = 0; - - if (serverCnxnFactory != null) { - numAliveConnections += serverCnxnFactory.getNumAliveConnections(); - } - - if (secureServerCnxnFactory != null) { - numAliveConnections += secureServerCnxnFactory.getNumAliveConnections(); - } - - return numAliveConnections; - } - - /** - * truncate the log to get in sync with others - * if in a quorum - * @param zxid the zxid that it needs to get in sync - * with others - * @throws IOException - */ - public void truncateLog(long zxid) throws IOException { - this.zkDb.truncateLog(zxid); - } - - public int getTickTime() { - return tickTime; - } - - public void setTickTime(int tickTime) { - LOG.info("tickTime set to {} ms", tickTime); - this.tickTime = tickTime; - } - - public static int getThrottledOpWaitTime() { - return throttledOpWaitTime; - } - - public static void setThrottledOpWaitTime(int time) { - LOG.info("throttledOpWaitTime set to {} ms", time); - throttledOpWaitTime = time; - } - - public int getMinSessionTimeout() { - return minSessionTimeout; - } - - public void setMinSessionTimeout(int min) { - this.minSessionTimeout = min == -1 ? tickTime * 2 : min; - LOG.info("minSessionTimeout set to {} ms", this.minSessionTimeout); - } - - public int getMaxSessionTimeout() { - return maxSessionTimeout; - } - - public void setMaxSessionTimeout(int max) { - this.maxSessionTimeout = max == -1 ? tickTime * 20 : max; - LOG.info("maxSessionTimeout set to {} ms", this.maxSessionTimeout); - } - - public int getClientPortListenBacklog() { - return listenBacklog; - } - - public void setClientPortListenBacklog(int backlog) { - this.listenBacklog = backlog; - LOG.info("clientPortListenBacklog set to {}", backlog); - } - - public int getClientPort() { - return serverCnxnFactory != null ? serverCnxnFactory.getLocalPort() : -1; - } - - public int getSecureClientPort() { - return secureServerCnxnFactory != null ? secureServerCnxnFactory.getLocalPort() : -1; - } - - /** Maximum number of connections allowed from particular host (ip) */ - public int getMaxClientCnxnsPerHost() { - if (serverCnxnFactory != null) { - return serverCnxnFactory.getMaxClientCnxnsPerHost(); - } - if (secureServerCnxnFactory != null) { - return secureServerCnxnFactory.getMaxClientCnxnsPerHost(); - } - return -1; - } - - public void setTxnLogFactory(FileTxnSnapLog txnLog) { - this.txnLogFactory = txnLog; - } - - public FileTxnSnapLog getTxnLogFactory() { - return this.txnLogFactory; - } - - /** - * Returns the elapsed sync of time of transaction log in milliseconds. - */ - public long getTxnLogElapsedSyncTime() { - return txnLogFactory.getTxnLogElapsedSyncTime(); - } - - public String getState() { - return "standalone"; - } - - public void dumpEphemerals(PrintWriter pwriter) { - zkDb.dumpEphemerals(pwriter); - } - - public Map<Long, Set<String>> getEphemerals() { - return zkDb.getEphemerals(); - } - - public double getConnectionDropChance() { - return connThrottle.getDropChance(); - } - - public void processConnectRequest(ServerCnxn cnxn, ConnectRequest request) throws IOException, ClientCnxnLimitException { - LOG.debug( - "Session establishment request from client {} client's lastZxid is 0x{}", - cnxn.getRemoteSocketAddress(), - Long.toHexString(request.getLastZxidSeen())); - - long sessionId = request.getSessionId(); - int tokensNeeded = 1; - if (connThrottle.isConnectionWeightEnabled()) { - if (sessionId == 0) { - if (localSessionEnabled) { - tokensNeeded = connThrottle.getRequiredTokensForLocal(); - } else { - tokensNeeded = connThrottle.getRequiredTokensForGlobal(); - } - } else { - tokensNeeded = connThrottle.getRequiredTokensForRenew(); - } - } - - if (!connThrottle.checkLimit(tokensNeeded)) { - throw new ClientCnxnLimitException(); - } - ServerMetrics.getMetrics().CONNECTION_TOKEN_DEFICIT.add(connThrottle.getDeficit()); - ServerMetrics.getMetrics().CONNECTION_REQUEST_COUNT.add(1); - - if (!cnxn.protocolManager.isReadonlyAvailable()) { - LOG.warn( - "Connection request from old client {}; will be dropped if server is in r-o mode", - cnxn.getRemoteSocketAddress()); - } - - if (!request.getReadOnly() && this instanceof ReadOnlyZooKeeperServer) { - String msg = "Refusing session request for not-read-only client " + cnxn.getRemoteSocketAddress(); - LOG.info(msg); - throw new CloseRequestException(msg, ServerCnxn.DisconnectReason.NOT_READ_ONLY_CLIENT); - } - if (request.getLastZxidSeen() > zkDb.dataTree.lastProcessedZxid) { - String msg = "Refusing session(0x" - + Long.toHexString(sessionId) - + ") request for client " - + cnxn.getRemoteSocketAddress() - + " as it has seen zxid 0x" - + Long.toHexString(request.getLastZxidSeen()) - + " our last zxid is 0x" - + Long.toHexString(getZKDatabase().getDataTreeLastProcessedZxid()) - + " client must try another server"; - - LOG.info(msg); - throw new CloseRequestException(msg, ServerCnxn.DisconnectReason.CLIENT_ZXID_AHEAD); - } - int sessionTimeout = request.getTimeOut(); - byte[] passwd = request.getPasswd(); - int minSessionTimeout = getMinSessionTimeout(); - if (sessionTimeout < minSessionTimeout) { - sessionTimeout = minSessionTimeout; - } - int maxSessionTimeout = getMaxSessionTimeout(); - if (sessionTimeout > maxSessionTimeout) { - sessionTimeout = maxSessionTimeout; - } - cnxn.setSessionTimeout(sessionTimeout); - // We don't want to receive any packets until we are sure that the - // session is setup - cnxn.disableRecv(); - if (sessionId == 0) { - long id = createSession(cnxn, passwd, sessionTimeout); - LOG.debug( - "Client attempting to establish new session: session = 0x{}, zxid = 0x{}, timeout = {}, address = {}", - Long.toHexString(id), - Long.toHexString(request.getLastZxidSeen()), - request.getTimeOut(), - cnxn.getRemoteSocketAddress()); - } else { - validateSession(cnxn, sessionId); - LOG.debug( - "Client attempting to renew session: session = 0x{}, zxid = 0x{}, timeout = {}, address = {}", - Long.toHexString(sessionId), - Long.toHexString(request.getLastZxidSeen()), - request.getTimeOut(), - cnxn.getRemoteSocketAddress()); - if (serverCnxnFactory != null) { - serverCnxnFactory.closeSession(sessionId, ServerCnxn.DisconnectReason.CLIENT_RECONNECT); - } - if (secureServerCnxnFactory != null) { - secureServerCnxnFactory.closeSession(sessionId, ServerCnxn.DisconnectReason.CLIENT_RECONNECT); - } - cnxn.setSessionId(sessionId); - reopenSession(cnxn, sessionId, passwd, sessionTimeout); - ServerMetrics.getMetrics().CONNECTION_REVALIDATE_COUNT.add(1); - - } - } - - /** - * Validate if a particular session can be reestablished. - * - * @param cnxn - * @param sessionId - */ - protected void validateSession(ServerCnxn cnxn, long sessionId) - throws IOException { - // do nothing - } - - public boolean shouldThrottle(long outStandingCount) { - int globalOutstandingLimit = getGlobalOutstandingLimit(); - if (globalOutstandingLimit < getInflight() || globalOutstandingLimit < getInProcess()) { - return outStandingCount > 0; - } - return false; - } - - long getFlushDelay() { - return flushDelay; - } - - static void setFlushDelay(long delay) { - LOG.info("{} = {} ms", FLUSH_DELAY, delay); - flushDelay = delay; - } - - long getMaxWriteQueuePollTime() { - return maxWriteQueuePollTime; - } - - static void setMaxWriteQueuePollTime(long maxTime) { - LOG.info("{} = {} ms", MAX_WRITE_QUEUE_POLL_SIZE, maxTime); - maxWriteQueuePollTime = maxTime; - } - - int getMaxBatchSize() { - return maxBatchSize; - } - - static void setMaxBatchSize(int size) { - LOG.info("{}={}", MAX_BATCH_SIZE, size); - maxBatchSize = size; - } - - private void initLargeRequestThrottlingSettings() { - setLargeRequestMaxBytes(Integer.getInteger("zookeeper.largeRequestMaxBytes", largeRequestMaxBytes)); - setLargeRequestThreshold(Integer.getInteger("zookeeper.largeRequestThreshold", -1)); - } - - public int getLargeRequestMaxBytes() { - return largeRequestMaxBytes; - } - - public void setLargeRequestMaxBytes(int bytes) { - if (bytes <= 0) { - LOG.warn("Invalid max bytes for all large requests {}. It should be a positive number.", bytes); - LOG.warn("Will not change the setting. The max bytes stay at {}", largeRequestMaxBytes); - } else { - largeRequestMaxBytes = bytes; - LOG.info("The max bytes for all large requests are set to {}", largeRequestMaxBytes); - } - } - - public int getLargeRequestThreshold() { - return largeRequestThreshold; - } - - public void setLargeRequestThreshold(int threshold) { - if (threshold == 0 || threshold < -1) { - LOG.warn("Invalid large request threshold {}. It should be -1 or positive. Setting to -1 ", threshold); - largeRequestThreshold = -1; - } else { - largeRequestThreshold = threshold; - LOG.info("The large request threshold is set to {}", largeRequestThreshold); - } - } - - public int getLargeRequestBytes() { - return currentLargeRequestBytes.get(); - } - - private boolean isLargeRequest(int length) { - // The large request limit is disabled when threshold is -1 - if (largeRequestThreshold == -1) { - return false; - } - return length > largeRequestThreshold; - } - - public boolean checkRequestSizeWhenReceivingMessage(int length) throws IOException { - if (!isLargeRequest(length)) { - return true; - } - if (currentLargeRequestBytes.get() + length <= largeRequestMaxBytes) { - return true; - } else { - ServerMetrics.getMetrics().LARGE_REQUESTS_REJECTED.add(1); - throw new IOException("Rejecting large request"); - } - - } - - private boolean checkRequestSizeWhenMessageReceived(int length) throws IOException { - if (!isLargeRequest(length)) { - return true; - } - - int bytes = currentLargeRequestBytes.addAndGet(length); - if (bytes > largeRequestMaxBytes) { - currentLargeRequestBytes.addAndGet(-length); - ServerMetrics.getMetrics().LARGE_REQUESTS_REJECTED.add(1); - throw new IOException("Rejecting large request"); - } - return true; - } - - public void requestFinished(Request request) { - int largeRequestLength = request.getLargeRequestSize(); - if (largeRequestLength != -1) { - currentLargeRequestBytes.addAndGet(-largeRequestLength); - } - } - - public void processPacket(ServerCnxn cnxn, RequestHeader h, RequestRecord request) throws IOException { - // Need to increase the outstanding request count first, otherwise - // there might be a race condition that it enabled recv after - // processing request and then disabled when check throttling. - // - // Be aware that we're actually checking the global outstanding - // request before this request. - // - // It's fine if the IOException thrown before we decrease the count - // in cnxn, since it will close the cnxn anyway. - cnxn.incrOutstandingAndCheckThrottle(h); - - if (h.getType() == OpCode.auth) { - LOG.info("got auth packet {}", cnxn.getRemoteSocketAddress()); - AuthPacket authPacket = request.readRecord(AuthPacket::new); - String scheme = authPacket.getScheme(); - ServerAuthenticationProvider ap = ProviderRegistry.getServerProvider(scheme); - Code authReturn = KeeperException.Code.AUTHFAILED; - if (ap != null) { - try { - // handleAuthentication may close the connection, to allow the client to choose - // a different server to connect to. - authReturn = ap.handleAuthentication( - new ServerAuthenticationProvider.ServerObjs(this, cnxn), - authPacket.getAuth()); - } catch (RuntimeException e) { - LOG.warn("Caught runtime exception from AuthenticationProvider: {}", scheme, e); - authReturn = KeeperException.Code.AUTHFAILED; - } - } - if (authReturn == KeeperException.Code.OK) { - LOG.info("Session 0x{}: auth success for scheme {} and address {}", - Long.toHexString(cnxn.getSessionId()), scheme, - cnxn.getRemoteSocketAddress()); - ReplyHeader rh = new ReplyHeader(h.getXid(), 0, KeeperException.Code.OK.intValue()); - cnxn.sendResponse(rh, null, null); - } else { - if (ap == null) { - LOG.warn( - "No authentication provider for scheme: {} has {}", - scheme, - ProviderRegistry.listProviders()); - } else { - LOG.warn("Authentication failed for scheme: {}", scheme); - } - // send a response... - ReplyHeader rh = new ReplyHeader(h.getXid(), 0, KeeperException.Code.AUTHFAILED.intValue()); - cnxn.sendResponse(rh, null, null); - // ... and close connection - cnxn.sendBuffer(ServerCnxnFactory.closeConn); - cnxn.disableRecv(); - } - return; - } else if (h.getType() == OpCode.sasl) { - processSasl(request, cnxn, h); - } else { - if (!authHelper.enforceAuthentication(cnxn, h.getXid())) { - // Authentication enforcement is failed - // Already sent response to user about failure and closed the session, lets return - return; - } else { - Request si = new Request(cnxn, cnxn.getSessionId(), h.getXid(), h.getType(), request, cnxn.getAuthInfo()); - int length = request.limit(); - if (isLargeRequest(length)) { - // checkRequestSize will throw IOException if request is rejected - checkRequestSizeWhenMessageReceived(length); - si.setLargeRequestSize(length); - } - si.setOwner(ServerCnxn.me); - submitRequest(si); - } - } - } - - private static boolean isSaslSuperUser(String id) { - if (id == null || id.isEmpty()) { - return false; - } - - Properties properties = System.getProperties(); - int prefixLen = SASL_SUPER_USER.length(); - - for (String k : properties.stringPropertyNames()) { - if (k.startsWith(SASL_SUPER_USER) - && (k.length() == prefixLen || k.charAt(prefixLen) == '.')) { - String value = properties.getProperty(k); - - if (value != null && value.equals(id)) { - return true; - } - } - } - - return false; - } - - private static boolean shouldAllowSaslFailedClientsConnect() { - return Boolean.getBoolean(ALLOW_SASL_FAILED_CLIENTS); - } - - private void processSasl(RequestRecord request, ServerCnxn cnxn, RequestHeader requestHeader) throws IOException { - LOG.debug("Responding to client SASL token."); - GetSASLRequest clientTokenRecord = request.readRecord(GetSASLRequest::new); - byte[] clientToken = clientTokenRecord.getToken(); - LOG.debug("Size of client SASL token: {}", clientToken.length); - byte[] responseToken = null; - try { - ZooKeeperSaslServer saslServer = cnxn.zooKeeperSaslServer; - try { - // note that clientToken might be empty (clientToken.length == 0): - // if using the DIGEST-MD5 mechanism, clientToken will be empty at the beginning of the - // SASL negotiation process. - responseToken = saslServer.evaluateResponse(clientToken); - if (saslServer.isComplete()) { - String authorizationID = saslServer.getAuthorizationID(); - LOG.info("Session 0x{}: adding SASL authorization for authorizationID: {}", - Long.toHexString(cnxn.getSessionId()), authorizationID); - cnxn.addAuthInfo(new Id("sasl", authorizationID)); - - if (isSaslSuperUser(authorizationID)) { - cnxn.addAuthInfo(new Id("super", "")); - LOG.info( - "Session 0x{}: Authenticated Id '{}' as super user", - Long.toHexString(cnxn.getSessionId()), - authorizationID); - } - } - } catch (SaslException e) { - LOG.warn("Client {} failed to SASL authenticate: {}", cnxn.getRemoteSocketAddress(), e); - if (shouldAllowSaslFailedClientsConnect() && !authHelper.isSaslAuthRequired()) { - LOG.warn("Maintaining client connection despite SASL authentication failure."); - } else { - int error; - if (authHelper.isSaslAuthRequired()) { - LOG.warn( - "Closing client connection due to server requires client SASL authenticaiton," - + "but client SASL authentication has failed, or client is not configured with SASL " - + "authentication."); - error = Code.SESSIONCLOSEDREQUIRESASLAUTH.intValue(); - } else { - LOG.warn("Closing client connection due to SASL authentication failure."); - error = Code.AUTHFAILED.intValue(); - } - - ReplyHeader replyHeader = new ReplyHeader(requestHeader.getXid(), 0, error); - cnxn.sendResponse(replyHeader, new SetSASLResponse(null), "response"); - cnxn.sendCloseSession(); - cnxn.disableRecv(); - return; - } - } - } catch (NullPointerException e) { - LOG.error("cnxn.saslServer is null: cnxn object did not initialize its saslServer properly."); - } - if (responseToken != null) { - LOG.debug("Size of server SASL response: {}", responseToken.length); - } - - ReplyHeader replyHeader = new ReplyHeader(requestHeader.getXid(), 0, Code.OK.intValue()); - Record record = new SetSASLResponse(responseToken); - cnxn.sendResponse(replyHeader, record, "response"); - } - - // entry point for quorum/Learner.java - public ProcessTxnResult processTxn(TxnHeader hdr, Record txn) { - processTxnForSessionEvents(null, hdr, txn); - return processTxnInDB(hdr, txn, null); - } - - // entry point for FinalRequestProcessor.java - public ProcessTxnResult processTxn(Request request) { - TxnHeader hdr = request.getHdr(); - processTxnForSessionEvents(request, hdr, request.getTxn()); - - final boolean writeRequest = (hdr != null); - final boolean quorumRequest = request.isQuorum(); - - // return fast w/o synchronization when we get a read - if (!writeRequest && !quorumRequest) { - return new ProcessTxnResult(); - } - synchronized (outstandingChanges) { - ProcessTxnResult rc = processTxnInDB(hdr, request.getTxn(), request.getTxnDigest()); - - // request.hdr is set for write requests, which are the only ones - // that add to outstandingChanges. - if (writeRequest) { - long zxid = hdr.getZxid(); - while (!outstandingChanges.isEmpty() - && outstandingChanges.peek().zxid <= zxid) { - ChangeRecord cr = outstandingChanges.remove(); - ServerMetrics.getMetrics().OUTSTANDING_CHANGES_REMOVED.add(1); - if (cr.zxid < zxid) { - LOG.warn( - "Zxid outstanding 0x{} is less than current 0x{}", - Long.toHexString(cr.zxid), - Long.toHexString(zxid)); - } - if (outstandingChangesForPath.get(cr.path) == cr) { - outstandingChangesForPath.remove(cr.path); - } - } - } - - // do not add non quorum packets to the queue. - if (quorumRequest) { - getZKDatabase().addCommittedProposal(request); - } - return rc; - } - } - - private void processTxnForSessionEvents(Request request, TxnHeader hdr, Record txn) { - int opCode = (request == null) ? hdr.getType() : request.type; - long sessionId = (request == null) ? hdr.getClientId() : request.sessionId; - - if (opCode == OpCode.createSession) { - if (hdr != null && txn instanceof CreateSessionTxn) { - CreateSessionTxn cst = (CreateSessionTxn) txn; - sessionTracker.commitSession(sessionId, cst.getTimeOut()); - } else if (request == null || !request.isLocalSession()) { - LOG.warn("*****>>>>> Got {} {}", txn.getClass(), txn.toString()); - } - } else if (opCode == OpCode.closeSession) { - sessionTracker.removeSession(sessionId); - } - } - - private ProcessTxnResult processTxnInDB(TxnHeader hdr, Record txn, TxnDigest digest) { - if (hdr == null) { - return new ProcessTxnResult(); - } else { - return getZKDatabase().processTxn(hdr, txn, digest); - } - } - - public Map<Long, Set<Long>> getSessionExpiryMap() { - return sessionTracker.getSessionExpiryMap(); - } - - /** - * This method is used to register the ZooKeeperServerShutdownHandler to get - * server's error or shutdown state change notifications. - * {@link ZooKeeperServerShutdownHandler#handle(State)} will be called for - * every server state changes {@link #setState(State)}. - * - * @param zkShutdownHandler shutdown handler - */ - void registerServerShutdownHandler(ZooKeeperServerShutdownHandler zkShutdownHandler) { - this.zkShutdownHandler = zkShutdownHandler; - } - - public boolean isResponseCachingEnabled() { - return isResponseCachingEnabled; - } - - public void setResponseCachingEnabled(boolean isEnabled) { - isResponseCachingEnabled = isEnabled; - } - - public ResponseCache getReadResponseCache() { - return isResponseCachingEnabled ? readResponseCache : null; - } - - public ResponseCache getGetChildrenResponseCache() { - return isResponseCachingEnabled ? getChildrenResponseCache : null; - } - - protected void registerMetrics() { - MetricsContext rootContext = ServerMetrics.getMetrics().getMetricsProvider().getRootContext(); - - final ZKDatabase zkdb = this.getZKDatabase(); - final ServerStats stats = this.serverStats(); - - rootContext.registerGauge("avg_latency", stats::getAvgLatency); - - rootContext.registerGauge("max_latency", stats::getMaxLatency); - rootContext.registerGauge("min_latency", stats::getMinLatency); - - rootContext.registerGauge("packets_received", stats::getPacketsReceived); - rootContext.registerGauge("packets_sent", stats::getPacketsSent); - rootContext.registerGauge("num_alive_connections", stats::getNumAliveClientConnections); - - rootContext.registerGauge("outstanding_requests", stats::getOutstandingRequests); - rootContext.registerGauge("uptime", stats::getUptime); - - rootContext.registerGauge("znode_count", zkdb::getNodeCount); - - rootContext.registerGauge("watch_count", zkdb.getDataTree()::getWatchCount); - rootContext.registerGauge("ephemerals_count", zkdb.getDataTree()::getEphemeralsCount); - - rootContext.registerGauge("approximate_data_size", zkdb.getDataTree()::cachedApproximateDataSize); - - rootContext.registerGauge("global_sessions", zkdb::getSessionCount); - rootContext.registerGauge("local_sessions", this.getSessionTracker()::getLocalSessionCount); - - OSMXBean osMbean = new OSMXBean(); - rootContext.registerGauge("open_file_descriptor_count", osMbean::getOpenFileDescriptorCount); - rootContext.registerGauge("max_file_descriptor_count", osMbean::getMaxFileDescriptorCount); - rootContext.registerGauge("connection_drop_probability", this::getConnectionDropChance); - - rootContext.registerGauge("last_client_response_size", stats.getClientResponseStats()::getLastBufferSize); - rootContext.registerGauge("max_client_response_size", stats.getClientResponseStats()::getMaxBufferSize); - rootContext.registerGauge("min_client_response_size", stats.getClientResponseStats()::getMinBufferSize); - - rootContext.registerGauge("outstanding_tls_handshake", this::getOutstandingHandshakeNum); - rootContext.registerGauge("auth_failed_count", stats::getAuthFailedCount); - rootContext.registerGauge("non_mtls_remote_conn_count", stats::getNonMTLSRemoteConnCount); - rootContext.registerGauge("non_mtls_local_conn_count", stats::getNonMTLSLocalConnCount); - - rootContext.registerGaugeSet(QuotaMetricsUtils.QUOTA_COUNT_LIMIT_PER_NAMESPACE, - () -> QuotaMetricsUtils.getQuotaCountLimit(zkDb.getDataTree())); - rootContext.registerGaugeSet(QuotaMetricsUtils.QUOTA_BYTES_LIMIT_PER_NAMESPACE, - () -> QuotaMetricsUtils.getQuotaBytesLimit(zkDb.getDataTree())); - rootContext.registerGaugeSet(QuotaMetricsUtils.QUOTA_COUNT_USAGE_PER_NAMESPACE, - () -> QuotaMetricsUtils.getQuotaCountUsage(zkDb.getDataTree())); - rootContext.registerGaugeSet(QuotaMetricsUtils.QUOTA_BYTES_USAGE_PER_NAMESPACE, - () -> QuotaMetricsUtils.getQuotaBytesUsage(zkDb.getDataTree())); - } - - protected void unregisterMetrics() { - - MetricsContext rootContext = ServerMetrics.getMetrics().getMetricsProvider().getRootContext(); - - rootContext.unregisterGauge("avg_latency"); - - rootContext.unregisterGauge("max_latency"); - rootContext.unregisterGauge("min_latency"); - - rootContext.unregisterGauge("packets_received"); - rootContext.unregisterGauge("packets_sent"); - rootContext.unregisterGauge("num_alive_connections"); - - rootContext.unregisterGauge("outstanding_requests"); - rootContext.unregisterGauge("uptime"); - - rootContext.unregisterGauge("znode_count"); - - rootContext.unregisterGauge("watch_count"); - rootContext.unregisterGauge("ephemerals_count"); - rootContext.unregisterGauge("approximate_data_size"); - - rootContext.unregisterGauge("global_sessions"); - rootContext.unregisterGauge("local_sessions"); - - rootContext.unregisterGauge("open_file_descriptor_count"); - rootContext.unregisterGauge("max_file_descriptor_count"); - rootContext.unregisterGauge("connection_drop_probability"); - - rootContext.unregisterGauge("last_client_response_size"); - rootContext.unregisterGauge("max_client_response_size"); - rootContext.unregisterGauge("min_client_response_size"); - - rootContext.unregisterGauge("auth_failed_count"); - rootContext.unregisterGauge("non_mtls_remote_conn_count"); - rootContext.unregisterGauge("non_mtls_local_conn_count"); - - rootContext.unregisterGaugeSet(QuotaMetricsUtils.QUOTA_COUNT_LIMIT_PER_NAMESPACE); - rootContext.unregisterGaugeSet(QuotaMetricsUtils.QUOTA_BYTES_LIMIT_PER_NAMESPACE); - rootContext.unregisterGaugeSet(QuotaMetricsUtils.QUOTA_COUNT_USAGE_PER_NAMESPACE); - rootContext.unregisterGaugeSet(QuotaMetricsUtils.QUOTA_BYTES_USAGE_PER_NAMESPACE); - } - - /** - * Hook into admin server, useful to expose additional data - * that do not represent metrics. - * - * @param response a sink which collects the data. - */ - public void dumpMonitorValues(BiConsumer<String, Object> response) { - ServerStats stats = serverStats(); - response.accept("version", Version.getFullVersion()); - response.accept("server_state", stats.getServerState()); - } - - /** - * Grant or deny authorization to an operation on a node as a function of: - * @param cnxn : the server connection or null for admin server commands - * @param acl : set of ACLs for the node - * @param perm : the permission that the client is requesting - * @param ids : the credentials supplied by the client - * @param path : the ZNode path - * @param setAcls : for set ACL operations, the list of ACLs being set. Otherwise null. - */ - public void checkACL(ServerCnxn cnxn, List<ACL> acl, int perm, List<Id> ids, String path, List<ACL> setAcls) throws KeeperException.NoAuthException { - if (skipACL) { - return; - } - - LOG.debug("Permission requested: {} ", perm); - LOG.debug("ACLs for node: {}", acl); - LOG.debug("Client credentials: {}", ids); - - if (acl == null || acl.size() == 0) { - return; - } - for (Id authId : ids) { - if (authId.getScheme().equals("super")) { - return; - } - } - for (ACL a : acl) { - Id id = a.getId(); - if ((a.getPerms() & perm) != 0) { - if (id.getScheme().equals("world") && id.getId().equals("anyone")) { - return; - } - ServerAuthenticationProvider ap = ProviderRegistry.getServerProvider(id.getScheme()); - if (ap != null) { - for (Id authId : ids) { - if (authId.getScheme().equals(id.getScheme()) - && ap.matches( - new ServerAuthenticationProvider.ServerObjs(this, cnxn), - new ServerAuthenticationProvider.MatchValues(path, authId.getId(), id.getId(), perm, setAcls))) { - return; - } - } - } - } - } - throw new KeeperException.NoAuthException(); - } - - /** - * check a path whether exceeded the quota. - * - * @param path - * the path of the node, used for the quota prefix check - * @param lastData - * the current node data, {@code null} for none - * @param data - * the data to be set, or {@code null} for none - * @param type - * currently, create and setData need to check quota - */ - public void checkQuota(String path, byte[] lastData, byte[] data, int type) throws KeeperException.QuotaExceededException { - if (!enforceQuota) { - return; - } - long dataBytes = (data == null) ? 0 : data.length; - ZKDatabase zkDatabase = getZKDatabase(); - String lastPrefix = zkDatabase.getDataTree().getMaxPrefixWithQuota(path); - if (StringUtils.isEmpty(lastPrefix)) { - return; - } - - final String namespace = PathUtils.getTopNamespace(path); - switch (type) { - case OpCode.create: - checkQuota(lastPrefix, dataBytes, 1, namespace); - break; - case OpCode.setData: - checkQuota(lastPrefix, dataBytes - (lastData == null ? 0 : lastData.length), 0, namespace); - break; - default: - throw new IllegalArgumentException("Unsupported OpCode for checkQuota: " + type); - } - } - - /** - * check a path whether exceeded the quota. - * - * @param lastPrefix - the path of the node which has a quota. - * @param bytesDiff - * the diff to be added to number of bytes - * @param countDiff - * the diff to be added to the count - * @param namespace - * the namespace for collecting quota exceeded errors - */ - private void checkQuota(String lastPrefix, long bytesDiff, long countDiff, String namespace) - throws KeeperException.QuotaExceededException { - LOG.debug("checkQuota: lastPrefix={}, bytesDiff={}, countDiff={}", lastPrefix, bytesDiff, countDiff); - - // now check the quota we set - String limitNode = Quotas.limitPath(lastPrefix); - DataNode node = getZKDatabase().getNode(limitNode); - StatsTrack limitStats; - if (node == null) { - // should not happen - LOG.error("Missing limit node for quota {}", limitNode); - return; - } - synchronized (node) { - limitStats = new StatsTrack(node.data); - } - //check the quota - boolean checkCountQuota = countDiff != 0 && (limitStats.getCount() > -1 || limitStats.getCountHardLimit() > -1); - boolean checkByteQuota = bytesDiff != 0 && (limitStats.getBytes() > -1 || limitStats.getByteHardLimit() > -1); - - if (!checkCountQuota && !checkByteQuota) { - return; - } - - //check the statPath quota - String statNode = Quotas.statPath(lastPrefix); - node = getZKDatabase().getNode(statNode); - - StatsTrack currentStats; - if (node == null) { - // should not happen - LOG.error("Missing node for stat {}", statNode); - return; - } - synchronized (node) { - currentStats = new StatsTrack(node.data); - } - - //check the Count Quota - if (checkCountQuota) { - long newCount = currentStats.getCount() + countDiff; - boolean isCountHardLimit = limitStats.getCountHardLimit() > -1; - long countLimit = isCountHardLimit ? limitStats.getCountHardLimit() : limitStats.getCount(); - - if (newCount > countLimit) { - String msg = "Quota exceeded: " + lastPrefix + " [current count=" + newCount + ", " + (isCountHardLimit ? "hard" : "soft") + "CountLimit=" + countLimit + "]"; - RATE_LOGGER.rateLimitLog(msg); - if (isCountHardLimit) { - updateQuotaExceededMetrics(namespace); - throw new KeeperException.QuotaExceededException(lastPrefix); - } - } - } - - //check the Byte Quota - if (checkByteQuota) { - long newBytes = currentStats.getBytes() + bytesDiff; - boolean isByteHardLimit = limitStats.getByteHardLimit() > -1; - long byteLimit = isByteHardLimit ? limitStats.getByteHardLimit() : limitStats.getBytes(); - if (newBytes > byteLimit) { - String msg = "Quota exceeded: " + lastPrefix + " [current bytes=" + newBytes + ", " + (isByteHardLimit ? "hard" : "soft") + "ByteLimit=" + byteLimit + "]"; - RATE_LOGGER.rateLimitLog(msg); - if (isByteHardLimit) { - updateQuotaExceededMetrics(namespace); - throw new KeeperException.QuotaExceededException(lastPrefix); - } - } - } - } - - public static boolean isDigestEnabled() { - return digestEnabled; - } - - public static void setDigestEnabled(boolean digestEnabled) { - LOG.info("{} = {}", ZOOKEEPER_DIGEST_ENABLED, digestEnabled); - ZooKeeperServer.digestEnabled = digestEnabled; - } - - public static boolean isSerializeLastProcessedZxidEnabled() { - return serializeLastProcessedZxidEnabled; - } - - public static void setSerializeLastProcessedZxidEnabled(boolean serializeLastZxidEnabled) { - serializeLastProcessedZxidEnabled = serializeLastZxidEnabled; - LOG.info("{} = {}", ZOOKEEPER_SERIALIZE_LAST_PROCESSED_ZXID_ENABLED, serializeLastZxidEnabled); - } - - /** - * Trim a path to get the immediate predecessor. - * - * @param path - * @return - * @throws KeeperException.BadArgumentsException - */ - private String parentPath(String path) throws KeeperException.BadArgumentsException { - int lastSlash = path.lastIndexOf('/'); - if (lastSlash == -1 || path.indexOf('\0') != -1 || getZKDatabase().isSpecialPath(path)) { - throw new KeeperException.BadArgumentsException(path); - } - return lastSlash == 0 ? "/" : path.substring(0, lastSlash); - } - - private String effectiveACLPath(Request request) throws KeeperException.BadArgumentsException, KeeperException.InvalidACLException { - boolean mustCheckACL = false; - String path = null; - List<ACL> acl = null; - - switch (request.type) { - case OpCode.create: - case OpCode.create2: { - CreateRequest req = request.readRequestRecordNoException(CreateRequest::new); - if (req != null) { - mustCheckACL = true; - acl = req.getAcl(); - path = parentPath(req.getPath()); - } - break; - } - case OpCode.delete: { - DeleteRequest req = request.readRequestRecordNoException(DeleteRequest::new); - if (req != null) { - path = parentPath(req.getPath()); - } - break; - } - case OpCode.setData: { - SetDataRequest req = request.readRequestRecordNoException(SetDataRequest::new); - if (req != null) { - path = req.getPath(); - } - break; - } - case OpCode.setACL: { - SetACLRequest req = request.readRequestRecordNoException(SetACLRequest::new); - if (req != null) { - mustCheckACL = true; - acl = req.getAcl(); - path = req.getPath(); - } - break; - } - } - - if (mustCheckACL) { - /* we ignore the extrapolated ACL returned by fixupACL because - * we only care about it being well-formed (and if it isn't, an - * exception will be raised). - */ - PrepRequestProcessor.fixupACL(path, request.authInfo, acl); - } - - return path; - } - - private int effectiveACLPerms(Request request) { - switch (request.type) { - case OpCode.create: - case OpCode.create2: - return ZooDefs.Perms.CREATE; - case OpCode.delete: - return ZooDefs.Perms.DELETE; - case OpCode.setData: - return ZooDefs.Perms.WRITE; - case OpCode.setACL: - return ZooDefs.Perms.ADMIN; - default: - return ZooDefs.Perms.ALL; - } - } - - /** - * Check Write Requests for Potential Access Restrictions - * <p> - * Before a request is being proposed to the quorum, lets check it - * against local ACLs. Non-write requests (read, session, etc.) - * are passed along. Invalid requests are sent a response. - * <p> - * While we are at it, if the request will set an ACL: make sure it's - * a valid one. - * - * @param request - * @return true if request is permitted, false if not. - */ - public boolean authWriteRequest(Request request) { - int err; - String pathToCheck; - - if (!enableEagerACLCheck) { - return true; - } - - err = KeeperException.Code.OK.intValue(); - - try { - pathToCheck = effectiveACLPath(request); - if (pathToCheck != null) { - checkACL(request.cnxn, zkDb.getACL(pathToCheck, null), effectiveACLPerms(request), request.authInfo, pathToCheck, null); - } - } catch (KeeperException.NoAuthException e) { - LOG.debug("Request failed ACL check", e); - err = e.code().intValue(); - } catch (KeeperException.InvalidACLException e) { - LOG.debug("Request has an invalid ACL check", e); - err = e.code().intValue(); - } catch (KeeperException.NoNodeException e) { - LOG.debug("ACL check against non-existent node: {}", e.getMessage()); - } catch (KeeperException.BadArgumentsException e) { - LOG.debug("ACL check against illegal node path: {}", e.getMessage()); - } catch (Throwable t) { - LOG.error("Uncaught exception in authWriteRequest with: ", t); - throw t; - } finally { - if (err != KeeperException.Code.OK.intValue()) { - /* This request has a bad ACL, so we are dismissing it early. */ - decInProcess(); - ReplyHeader rh = new ReplyHeader(request.cxid, 0, err); - try { - request.cnxn.sendResponse(rh, null, null); - } catch (IOException e) { - LOG.error("IOException : {}", e); - } - } - } - - return err == KeeperException.Code.OK.intValue(); - } - - public int getOutstandingHandshakeNum() { - if (serverCnxnFactory instanceof NettyServerCnxnFactory) { - return ((NettyServerCnxnFactory) serverCnxnFactory).getOutstandingHandshakeNum(); - } else { - return 0; - } - } - - public boolean isReconfigEnabled() { - return this.reconfigEnabled; - } - - public ZooKeeperServerShutdownHandler getZkShutdownHandler() { - return zkShutdownHandler; - } - - static void updateQuotaExceededMetrics(final String namespace) { - if (namespace == null) { - return; - } - ServerMetrics.getMetrics().QUOTA_EXCEEDED_ERROR_PER_NAMESPACE.add(namespace, 1); - } -} - diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/LeaderZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/LeaderZooKeeperServer.java deleted file mode 100644 index 1f629bed73d..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/LeaderZooKeeperServer.java +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.quorum; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import javax.management.JMException; -import org.apache.zookeeper.KeeperException.SessionExpiredException; -import org.apache.zookeeper.jmx.MBeanRegistry; -import org.apache.zookeeper.metrics.MetricsContext; -import org.apache.zookeeper.server.ContainerManager; -import org.apache.zookeeper.server.DataTreeBean; -import org.apache.zookeeper.server.FinalRequestProcessor; -import org.apache.zookeeper.server.PrepRequestProcessor; -import org.apache.zookeeper.server.Request; -import org.apache.zookeeper.server.RequestProcessor; -import org.apache.zookeeper.server.ServerCnxn; -import org.apache.zookeeper.server.ServerMetrics; -import org.apache.zookeeper.server.ZKDatabase; -import org.apache.zookeeper.server.persistence.FileTxnSnapLog; - -/** - * - * Just like the standard ZooKeeperServer. We just replace the request - * processors: PrepRequestProcessor -> ProposalRequestProcessor -> - * CommitProcessor -> Leader.ToBeAppliedRequestProcessor -> - * FinalRequestProcessor - */ -public class LeaderZooKeeperServer extends QuorumZooKeeperServer { - - private ContainerManager containerManager; // guarded by sync - - CommitProcessor commitProcessor; - - PrepRequestProcessor prepRequestProcessor; - - /** - * @throws IOException - */ - public LeaderZooKeeperServer(FileTxnSnapLog logFactory, QuorumPeer self, ZKDatabase zkDb) throws IOException { - super(logFactory, self.tickTime, self.minSessionTimeout, self.maxSessionTimeout, self.clientPortListenBacklog, zkDb, self); - } - - public Leader getLeader() { - return self.leader; - } - - @Override - protected void setupRequestProcessors() { - RequestProcessor finalProcessor = new FinalRequestProcessor(this); - RequestProcessor toBeAppliedProcessor = new Leader.ToBeAppliedRequestProcessor(finalProcessor, getLeader()); - commitProcessor = new CommitProcessor(toBeAppliedProcessor, Long.toString(getServerId()), false, getZooKeeperServerListener()); - commitProcessor.start(); - ProposalRequestProcessor proposalProcessor = new ProposalRequestProcessor(this, commitProcessor); - proposalProcessor.initialize(); - prepRequestProcessor = new PrepRequestProcessor(this, proposalProcessor); - prepRequestProcessor.start(); - firstProcessor = new LeaderRequestProcessor(this, prepRequestProcessor); - - setupContainerManager(); - } - - private synchronized void setupContainerManager() { - containerManager = new ContainerManager( - getZKDatabase(), - prepRequestProcessor, - Integer.getInteger("znode.container.checkIntervalMs", (int) TimeUnit.MINUTES.toMillis(1)), - Integer.getInteger("znode.container.maxPerMinute", 10000), - Long.getLong("znode.container.maxNeverUsedIntervalMs", 0) - ); - } - - @Override - public synchronized void startup() { - super.startup(); - if (containerManager != null) { - containerManager.start(); - } - } - - @Override - protected void registerMetrics() { - super.registerMetrics(); - - MetricsContext rootContext = ServerMetrics.getMetrics().getMetricsProvider().getRootContext(); - rootContext.registerGauge("learners", gaugeWithLeader( - (leader) -> leader.getLearners().size()) - ); - rootContext.registerGauge("synced_followers", gaugeWithLeader( - (leader) -> leader.getForwardingFollowers().size() - )); - rootContext.registerGauge("synced_non_voting_followers", gaugeWithLeader( - (leader) -> leader.getNonVotingFollowers().size() - )); - rootContext.registerGauge("synced_observers", self::getSynced_observers_metric); - rootContext.registerGauge("pending_syncs", gaugeWithLeader( - (leader) -> leader.getNumPendingSyncs() - )); - rootContext.registerGauge("leader_uptime", gaugeWithLeader( - (leader) -> leader.getUptime() - )); - rootContext.registerGauge("last_proposal_size", gaugeWithLeader( - (leader) -> leader.getProposalStats().getLastBufferSize() - )); - rootContext.registerGauge("max_proposal_size", gaugeWithLeader( - (leader) -> leader.getProposalStats().getMaxBufferSize() - )); - rootContext.registerGauge("min_proposal_size", gaugeWithLeader( - (leader) -> leader.getProposalStats().getMinBufferSize() - )); - } - - private org.apache.zookeeper.metrics.Gauge gaugeWithLeader(Function<Leader, Number> supplier) { - return () -> { - final Leader leader = getLeader(); - if (leader == null) { - return null; - } - return supplier.apply(leader); - }; - } - - @Override - protected void unregisterMetrics() { - super.unregisterMetrics(); - - MetricsContext rootContext = ServerMetrics.getMetrics().getMetricsProvider().getRootContext(); - rootContext.unregisterGauge("learners"); - rootContext.unregisterGauge("synced_followers"); - rootContext.unregisterGauge("synced_non_voting_followers"); - rootContext.unregisterGauge("synced_observers"); - rootContext.unregisterGauge("pending_syncs"); - rootContext.unregisterGauge("leader_uptime"); - - rootContext.unregisterGauge("last_proposal_size"); - rootContext.unregisterGauge("max_proposal_size"); - rootContext.unregisterGauge("min_proposal_size"); - } - - @Override - public synchronized void shutdown(boolean fullyShutDown) { - if (containerManager != null) { - containerManager.stop(); - } - super.shutdown(fullyShutDown); - } - - @Override - public int getGlobalOutstandingLimit() { - int divisor = self.getQuorumSize() > 2 ? self.getQuorumSize() - 1 : 1; - int globalOutstandingLimit = super.getGlobalOutstandingLimit() / divisor; - return globalOutstandingLimit; - } - - @Override - public void createSessionTracker() { - sessionTracker = new LeaderSessionTracker( - this, - getZKDatabase().getSessionWithTimeOuts(), - tickTime, - self.getMyId(), - self.areLocalSessionsEnabled(), - getZooKeeperServerListener()); - } - - public boolean touch(long sess, int to) { - return sessionTracker.touchSession(sess, to); - } - - public boolean checkIfValidGlobalSession(long sess, int to) { - if (self.areLocalSessionsEnabled() && !upgradeableSessionTracker.isGlobalSession(sess)) { - return false; - } - return sessionTracker.touchSession(sess, to); - } - - /** - * Requests coming from the learner should go directly to - * PrepRequestProcessor - * - * @param request - */ - public void submitLearnerRequest(Request request) { - /* - * Requests coming from the learner should have gone through - * submitRequest() on each server which already perform some request - * validation, so we don't need to do it again. - * - * Additionally, LearnerHandler should start submitting requests into - * the leader's pipeline only when the leader's server is started, so we - * can submit the request directly into PrepRequestProcessor. - * - * This is done so that requests from learners won't go through - * LeaderRequestProcessor which perform local session upgrade. - */ - prepRequestProcessor.processRequest(request); - } - - @Override - protected void registerJMX() { - // register with JMX - try { - jmxDataTreeBean = new DataTreeBean(getZKDatabase().getDataTree()); - MBeanRegistry.getInstance().register(jmxDataTreeBean, jmxServerBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - jmxDataTreeBean = null; - } - } - - public void registerJMX(LeaderBean leaderBean, LocalPeerBean localPeerBean) { - // register with JMX - if (self.jmxLeaderElectionBean != null) { - try { - MBeanRegistry.getInstance().unregister(self.jmxLeaderElectionBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - } - self.jmxLeaderElectionBean = null; - } - - try { - jmxServerBean = leaderBean; - MBeanRegistry.getInstance().register(leaderBean, localPeerBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - jmxServerBean = null; - } - } - - boolean registerJMX(LearnerHandlerBean handlerBean) { - try { - MBeanRegistry.getInstance().register(handlerBean, jmxServerBean); - return true; - } catch (JMException e) { - LOG.warn("Could not register connection", e); - } - return false; - } - - @Override - protected void unregisterJMX() { - // unregister from JMX - try { - if (jmxDataTreeBean != null) { - MBeanRegistry.getInstance().unregister(jmxDataTreeBean); - } - } catch (Exception e) { - LOG.warn("Failed to unregister with JMX", e); - } - jmxDataTreeBean = null; - } - - protected void unregisterJMX(Leader leader) { - // unregister from JMX - try { - if (jmxServerBean != null) { - MBeanRegistry.getInstance().unregister(jmxServerBean); - } - } catch (Exception e) { - LOG.warn("Failed to unregister with JMX", e); - } - jmxServerBean = null; - } - - @Override - public String getState() { - return "leader"; - } - - /** - * Returns the id of the associated QuorumPeer, which will do for a unique - * id of this server. - */ - @Override - public long getServerId() { - return self.getMyId(); - } - - @Override - protected void revalidateSession(ServerCnxn cnxn, long sessionId, int sessionTimeout) throws IOException { - super.revalidateSession(cnxn, sessionId, sessionTimeout); - try { - // setowner as the leader itself, unless updated - // via the follower handlers - setOwner(sessionId, ServerCnxn.me); - } catch (SessionExpiredException e) { - // this is ok, it just means that the session revalidation failed. - } - } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/Learner.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/Learner.java deleted file mode 100644 index 3c7b2148400..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/Learner.java +++ /dev/null @@ -1,928 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.quorum; - -import static java.nio.charset.StandardCharsets.UTF_8; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import javax.net.ssl.SSLSocket; -import org.apache.jute.BinaryInputArchive; -import org.apache.jute.BinaryOutputArchive; -import org.apache.jute.InputArchive; -import org.apache.jute.OutputArchive; -import org.apache.jute.Record; -import org.apache.zookeeper.ZooDefs.OpCode; -import org.apache.zookeeper.common.Time; -import org.apache.zookeeper.common.X509Exception; -import org.apache.zookeeper.server.ExitCode; -import org.apache.zookeeper.server.Request; -import org.apache.zookeeper.server.ServerCnxn; -import org.apache.zookeeper.server.ServerMetrics; -import org.apache.zookeeper.server.TxnLogEntry; -import org.apache.zookeeper.server.ZooTrace; -import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer; -import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; -import org.apache.zookeeper.server.util.ConfigUtils; -import org.apache.zookeeper.server.util.MessageTracker; -import org.apache.zookeeper.server.util.SerializeUtils; -import org.apache.zookeeper.server.util.ZxidUtils; -import org.apache.zookeeper.txn.SetDataTxn; -import org.apache.zookeeper.txn.TxnDigest; -import org.apache.zookeeper.txn.TxnHeader; -import org.apache.zookeeper.util.ServiceUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class is the superclass of two of the three main actors in a ZK - * ensemble: Followers and Observers. Both Followers and Observers share - * a good deal of code which is moved into Peer to avoid duplication. - */ -public class Learner { - - static class PacketInFlight { - - TxnHeader hdr; - Record rec; - TxnDigest digest; - - } - - QuorumPeer self; - LearnerZooKeeperServer zk; - - protected BufferedOutputStream bufferedOutput; - - protected Socket sock; - protected MultipleAddresses leaderAddr; - protected AtomicBoolean sockBeingClosed = new AtomicBoolean(false); - - /** - * Socket getter - */ - public Socket getSocket() { - return sock; - } - - LearnerSender sender = null; - protected InputArchive leaderIs; - protected OutputArchive leaderOs; - /** the protocol version of the leader */ - protected int leaderProtocolVersion = 0x01; - - private static final int BUFFERED_MESSAGE_SIZE = 10; - protected final MessageTracker messageTracker = new MessageTracker(BUFFERED_MESSAGE_SIZE); - - protected static final Logger LOG = LoggerFactory.getLogger(Learner.class); - - /** - * Time to wait after connection attempt with the Leader or LearnerMaster before this - * Learner tries to connect again. - */ - private static final int leaderConnectDelayDuringRetryMs = Integer.getInteger("zookeeper.leaderConnectDelayDuringRetryMs", 100); - - private static final boolean nodelay = System.getProperty("follower.nodelay", "true").equals("true"); - - public static final String LEARNER_ASYNC_SENDING = "zookeeper.learner.asyncSending"; - private static boolean asyncSending = - Boolean.parseBoolean(ConfigUtils.getPropertyBackwardCompatibleWay(LEARNER_ASYNC_SENDING)); - public static final String LEARNER_CLOSE_SOCKET_ASYNC = "zookeeper.learner.closeSocketAsync"; - public static final boolean closeSocketAsync = Boolean - .parseBoolean(ConfigUtils.getPropertyBackwardCompatibleWay(LEARNER_CLOSE_SOCKET_ASYNC)); - - static { - LOG.info("leaderConnectDelayDuringRetryMs: {}", leaderConnectDelayDuringRetryMs); - LOG.info("TCP NoDelay set to: {}", nodelay); - LOG.info("{} = {}", LEARNER_ASYNC_SENDING, asyncSending); - LOG.info("{} = {}", LEARNER_CLOSE_SOCKET_ASYNC, closeSocketAsync); - } - - final ConcurrentHashMap<Long, ServerCnxn> pendingRevalidations = new ConcurrentHashMap<>(); - - public int getPendingRevalidationsCount() { - return pendingRevalidations.size(); - } - - // for testing - protected static void setAsyncSending(boolean newMode) { - asyncSending = newMode; - LOG.info("{} = {}", LEARNER_ASYNC_SENDING, asyncSending); - - } - protected static boolean getAsyncSending() { - return asyncSending; - } - /** - * validate a session for a client - * - * @param clientId - * the client to be revalidated - * @param timeout - * the timeout for which the session is valid - * @throws IOException - */ - void validateSession(ServerCnxn cnxn, long clientId, int timeout) throws IOException { - LOG.info("Revalidating client: 0x{}", Long.toHexString(clientId)); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(baos); - dos.writeLong(clientId); - dos.writeInt(timeout); - dos.close(); - QuorumPacket qp = new QuorumPacket(Leader.REVALIDATE, -1, baos.toByteArray(), null); - pendingRevalidations.put(clientId, cnxn); - if (LOG.isTraceEnabled()) { - ZooTrace.logTraceMessage( - LOG, - ZooTrace.SESSION_TRACE_MASK, - "To validate session 0x" + Long.toHexString(clientId)); - } - writePacket(qp, true); - } - - /** - * write a packet to the leader. - * - * This method is called by multiple threads. We need to make sure that only one thread is writing to leaderOs at a time. - * When packets are sent synchronously, writing is done within a synchronization block. - * When packets are sent asynchronously, sender.queuePacket() is called, which writes to a BlockingQueue, which is thread-safe. - * Reading from this BlockingQueue and writing to leaderOs is the learner sender thread only. - * So we have only one thread writing to leaderOs at a time in either case. - * - * @param pp - * the proposal packet to be sent to the leader - * @throws IOException - */ - void writePacket(QuorumPacket pp, boolean flush) throws IOException { - if (asyncSending) { - sender.queuePacket(pp); - } else { - writePacketNow(pp, flush); - } - } - - void writePacketNow(QuorumPacket pp, boolean flush) throws IOException { - synchronized (leaderOs) { - if (pp != null) { - messageTracker.trackSent(pp.getType()); - leaderOs.writeRecord(pp, "packet"); - } - if (flush) { - bufferedOutput.flush(); - } - } - } - - /** - * Start thread that will forward any packet in the queue to the leader - */ - protected void startSendingThread() { - sender = new LearnerSender(this); - sender.start(); - } - - /** - * read a packet from the leader - * - * @param pp - * the packet to be instantiated - * @throws IOException - */ - void readPacket(QuorumPacket pp) throws IOException { - synchronized (leaderIs) { - leaderIs.readRecord(pp, "packet"); - messageTracker.trackReceived(pp.getType()); - } - if (LOG.isTraceEnabled()) { - final long traceMask = - (pp.getType() == Leader.PING) ? ZooTrace.SERVER_PING_TRACE_MASK - : ZooTrace.SERVER_PACKET_TRACE_MASK; - - ZooTrace.logQuorumPacket(LOG, traceMask, 'i', pp); - } - } - - /** - * send a request packet to the leader - * - * @param request - * the request from the client - * @throws IOException - */ - void request(Request request) throws IOException { - if (request.isThrottled()) { - LOG.error("Throttled request sent to leader: {}. Exiting", request); - ServiceUtils.requestSystemExit(ExitCode.UNEXPECTED_ERROR.getValue()); - } - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream oa = new DataOutputStream(baos); - oa.writeLong(request.sessionId); - oa.writeInt(request.cxid); - oa.writeInt(request.type); - byte[] payload = request.readRequestBytes(); - if (payload != null) { - oa.write(payload); - } - oa.close(); - QuorumPacket qp = new QuorumPacket(Leader.REQUEST, -1, baos.toByteArray(), request.authInfo); - writePacket(qp, true); - } - - /** - * Returns the address of the node we think is the leader. - */ - protected QuorumServer findLeader() { - QuorumServer leaderServer = null; - // Find the leader by id - Vote current = self.getCurrentVote(); - for (QuorumServer s : self.getView().values()) { - if (s.id == current.getId()) { - // Ensure we have the leader's correct IP address before - // attempting to connect. - s.recreateSocketAddresses(); - leaderServer = s; - break; - } - } - if (leaderServer == null) { - LOG.warn("Couldn't find the leader with id = {}", current.getId()); - } - return leaderServer; - } - - /** - * Overridable helper method to return the System.nanoTime(). - * This method behaves identical to System.nanoTime(). - */ - protected long nanoTime() { - return System.nanoTime(); - } - - /** - * Overridable helper method to simply call sock.connect(). This can be - * overriden in tests to fake connection success/failure for connectToLeader. - */ - protected void sockConnect(Socket sock, InetSocketAddress addr, int timeout) throws IOException { - sock.connect(addr, timeout); - } - - /** - * Establish a connection with the LearnerMaster found by findLearnerMaster. - * Followers only connect to Leaders, Observers can connect to any active LearnerMaster. - * Retries until either initLimit time has elapsed or 5 tries have happened. - * @param multiAddr - the address of the Peer to connect to. - * @throws IOException - if the socket connection fails on the 5th attempt - * if there is an authentication failure while connecting to leader - */ - protected void connectToLeader(MultipleAddresses multiAddr, String hostname) throws IOException { - - this.leaderAddr = multiAddr; - Set<InetSocketAddress> addresses; - if (self.isMultiAddressReachabilityCheckEnabled()) { - // even if none of the addresses are reachable, we want to try to establish connection - // see ZOOKEEPER-3758 - addresses = multiAddr.getAllReachableAddressesOrAll(); - } else { - addresses = multiAddr.getAllAddresses(); - } - ExecutorService executor = Executors.newFixedThreadPool(addresses.size()); - CountDownLatch latch = new CountDownLatch(addresses.size()); - AtomicReference<Socket> socket = new AtomicReference<>(null); - addresses.stream().map(address -> new LeaderConnector(address, socket, latch)).forEach(executor::submit); - - try { - latch.await(); - } catch (InterruptedException e) { - LOG.warn("Interrupted while trying to connect to Leader", e); - } finally { - executor.shutdown(); - try { - if (!executor.awaitTermination(1, TimeUnit.SECONDS)) { - LOG.error("not all the LeaderConnector terminated properly"); - } - } catch (InterruptedException ie) { - LOG.error("Interrupted while terminating LeaderConnector executor.", ie); - } - } - - if (socket.get() == null) { - throw new IOException("Failed connect to " + multiAddr); - } else { - sock = socket.get(); - sockBeingClosed.set(false); - } - - self.authLearner.authenticate(sock, hostname); - - leaderIs = BinaryInputArchive.getArchive(new BufferedInputStream(sock.getInputStream())); - bufferedOutput = new BufferedOutputStream(sock.getOutputStream()); - leaderOs = BinaryOutputArchive.getArchive(bufferedOutput); - if (asyncSending) { - startSendingThread(); - } - } - - class LeaderConnector implements Runnable { - - private AtomicReference<Socket> socket; - private InetSocketAddress address; - private CountDownLatch latch; - - LeaderConnector(InetSocketAddress address, AtomicReference<Socket> socket, CountDownLatch latch) { - this.address = address; - this.socket = socket; - this.latch = latch; - } - - @Override - public void run() { - try { - Thread.currentThread().setName("LeaderConnector-" + address); - Socket sock = connectToLeader(); - - if (sock != null && sock.isConnected()) { - if (socket.compareAndSet(null, sock)) { - LOG.info("Successfully connected to leader, using address: {}", address); - } else { - LOG.info("Connection to the leader is already established, close the redundant connection"); - sock.close(); - } - } - - } catch (Exception e) { - LOG.error("Failed connect to {}", address, e); - } finally { - latch.countDown(); - } - } - - private Socket connectToLeader() throws IOException, X509Exception, InterruptedException { - Socket sock = createSocket(); - - // leader connection timeout defaults to tickTime * initLimit - int connectTimeout = self.tickTime * self.initLimit; - - // but if connectToLearnerMasterLimit is specified, use that value to calculate - // timeout instead of using the initLimit value - if (self.connectToLearnerMasterLimit > 0) { - connectTimeout = self.tickTime * self.connectToLearnerMasterLimit; - } - - int remainingTimeout; - long startNanoTime = nanoTime(); - - for (int tries = 0; tries < 5 && socket.get() == null; tries++) { - try { - // recalculate the init limit time because retries sleep for 1000 milliseconds - remainingTimeout = connectTimeout - (int) ((nanoTime() - startNanoTime) / 1_000_000); - if (remainingTimeout <= 0) { - LOG.error("connectToLeader exceeded on retries."); - throw new IOException("connectToLeader exceeded on retries."); - } - - sockConnect(sock, address, Math.min(connectTimeout, remainingTimeout)); - if (self.isSslQuorum()) { - ((SSLSocket) sock).startHandshake(); - } - sock.setTcpNoDelay(nodelay); - break; - } catch (IOException e) { - remainingTimeout = connectTimeout - (int) ((nanoTime() - startNanoTime) / 1_000_000); - - if (remainingTimeout <= leaderConnectDelayDuringRetryMs) { - LOG.error( - "Unexpected exception, connectToLeader exceeded. tries={}, remaining init limit={}, connecting to {}", - tries, - remainingTimeout, - address, - e); - throw e; - } else if (tries >= 4) { - LOG.error( - "Unexpected exception, retries exceeded. tries={}, remaining init limit={}, connecting to {}", - tries, - remainingTimeout, - address, - e); - throw e; - } else { - LOG.warn( - "Unexpected exception, tries={}, remaining init limit={}, connecting to {}", - tries, - remainingTimeout, - address, - e); - sock = createSocket(); - } - } - Thread.sleep(leaderConnectDelayDuringRetryMs); - } - - return sock; - } - } - - /** - * Creating a simple or and SSL socket. - * This can be overridden in tests to fake already connected sockets for connectToLeader. - */ - protected Socket createSocket() throws X509Exception, IOException { - Socket sock; - if (self.isSslQuorum()) { - sock = self.getX509Util().createSSLSocket(); - } else { - sock = new Socket(); - } - sock.setSoTimeout(self.tickTime * self.initLimit); - return sock; - } - - /** - * Once connected to the leader or learner master, perform the handshake - * protocol to establish a following / observing connection. - * @param pktType - * @return the zxid the Leader sends for synchronization purposes. - * @throws IOException - */ - protected long registerWithLeader(int pktType) throws IOException { - /* - * Send follower info, including last zxid and sid - */ - long lastLoggedZxid = self.getLastLoggedZxid(); - QuorumPacket qp = new QuorumPacket(); - qp.setType(pktType); - qp.setZxid(ZxidUtils.makeZxid(self.getAcceptedEpoch(), 0)); - - /* - * Add sid to payload - */ - LearnerInfo li = new LearnerInfo(self.getMyId(), 0x10000, self.getQuorumVerifier().getVersion()); - ByteArrayOutputStream bsid = new ByteArrayOutputStream(); - BinaryOutputArchive boa = BinaryOutputArchive.getArchive(bsid); - boa.writeRecord(li, "LearnerInfo"); - qp.setData(bsid.toByteArray()); - - writePacket(qp, true); - readPacket(qp); - final long newEpoch = ZxidUtils.getEpochFromZxid(qp.getZxid()); - if (qp.getType() == Leader.LEADERINFO) { - // we are connected to a 1.0 server so accept the new epoch and read the next packet - leaderProtocolVersion = ByteBuffer.wrap(qp.getData()).getInt(); - byte[] epochBytes = new byte[4]; - final ByteBuffer wrappedEpochBytes = ByteBuffer.wrap(epochBytes); - if (newEpoch > self.getAcceptedEpoch()) { - wrappedEpochBytes.putInt((int) self.getCurrentEpoch()); - self.setAcceptedEpoch(newEpoch); - } else if (newEpoch == self.getAcceptedEpoch()) { - // since we have already acked an epoch equal to the leaders, we cannot ack - // again, but we still need to send our lastZxid to the leader so that we can - // sync with it if it does assume leadership of the epoch. - // the -1 indicates that this reply should not count as an ack for the new epoch - wrappedEpochBytes.putInt(-1); - } else { - throw new IOException("Leaders epoch, " - + newEpoch - + " is less than accepted epoch, " - + self.getAcceptedEpoch()); - } - QuorumPacket ackNewEpoch = new QuorumPacket(Leader.ACKEPOCH, lastLoggedZxid, epochBytes, null); - writePacket(ackNewEpoch, true); - return ZxidUtils.makeZxid(newEpoch, 0); - } else { - if (newEpoch > self.getAcceptedEpoch()) { - self.setAcceptedEpoch(newEpoch); - } - if (qp.getType() != Leader.NEWLEADER) { - LOG.error("First packet should have been NEWLEADER"); - throw new IOException("First packet should have been NEWLEADER"); - } - return qp.getZxid(); - } - } - - /** - * Finally, synchronize our history with the Leader (if Follower) - * or the LearnerMaster (if Observer). - * @param newLeaderZxid - * @throws IOException - * @throws InterruptedException - */ - protected void syncWithLeader(long newLeaderZxid) throws Exception { - long newEpoch = ZxidUtils.getEpochFromZxid(newLeaderZxid); - QuorumVerifier newLeaderQV = null; - - class SyncHelper { - - // In the DIFF case we don't need to do a snapshot because the transactions will sync on top of any existing snapshot. - // For SNAP and TRUNC the snapshot is needed to save that history. - boolean willSnapshot = true; - boolean syncSnapshot = false; - - // PROPOSALs received during sync, for matching up with COMMITs. - Deque<PacketInFlight> proposals = new ArrayDeque<>(); - - // PROPOSALs we delay forwarding to the ZK server until sync is done. - Deque<PacketInFlight> delayedProposals = new ArrayDeque<>(); - - // COMMITs we delay forwarding to the ZK server until sync is done. - Deque<Long> delayedCommits = new ArrayDeque<>(); - - void syncSnapshot() { - syncSnapshot = true; - } - - void noSnapshot() { - willSnapshot = false; - } - - void propose(PacketInFlight pif) { - proposals.add(pif); - delayedProposals.add(pif); - } - - PacketInFlight nextProposal() { - return proposals.peekFirst(); - } - - void commit() { - PacketInFlight packet = proposals.remove(); - if (willSnapshot) { - zk.processTxn(packet.hdr, packet.rec); - delayedProposals.remove(); - } else { - delayedCommits.add(packet.hdr.getZxid()); - } - } - - void writeState() throws IOException, InterruptedException { - // Ensure all received transaction PROPOSALs are written before we ACK the NEWLEADER, - // since this allows the leader to apply those transactions to its served state: - if (willSnapshot) { - zk.takeSnapshot(syncSnapshot); // either, the snapshot contains the transactions, - willSnapshot = false; // but anything after this needs to go to the transaction log; or - } - - sock.setSoTimeout(self.tickTime * self.syncLimit); - self.setSyncMode(QuorumPeer.SyncMode.NONE); - zk.startupWithoutServing(); - - // if we're a follower, we need to ensure the transactions are safely logged before ACK'ing. - if (zk instanceof FollowerZooKeeperServer) { - FollowerZooKeeperServer fzk = (FollowerZooKeeperServer) zk; - // The leader expects the NEWLEADER ACK to precede all the PROPOSAL ACKs, so we only write them first. - fzk.syncProcessor.setDelayForwarding(true); - for (PacketInFlight p : delayedProposals) { - fzk.logRequest(p.hdr, p.rec, p.digest); - } - delayedProposals.clear(); - fzk.syncProcessor.syncFlush(); - } - - self.setCurrentEpoch(newEpoch); - } - - void flushAcks() throws InterruptedException { - if (zk instanceof FollowerZooKeeperServer) { - // The NEWLEADER is ACK'ed, and we can now ACK the PROPOSALs we wrote in writeState. - FollowerZooKeeperServer fzk = (FollowerZooKeeperServer) zk; - fzk.syncProcessor.setDelayForwarding(false); - fzk.syncProcessor.syncFlush(); // Ensure these are all ACK'ed before the UPTODATE ACK. - } - } - - void applyDelayedPackets() { - // Any delayed packets must now be applied: all PROPOSALs first, then any COMMITs. - if (zk instanceof FollowerZooKeeperServer) { - FollowerZooKeeperServer fzk = (FollowerZooKeeperServer) zk; - for (PacketInFlight p : delayedProposals) { - fzk.logRequest(p.hdr, p.rec, p.digest); - } - for (Long zxid : delayedCommits) { - fzk.commit(zxid); - } - } else if (zk instanceof ObserverZooKeeperServer) { - ObserverZooKeeperServer ozk = (ObserverZooKeeperServer) zk; - for (PacketInFlight p : delayedProposals) { - Long zxid = delayedCommits.peekFirst(); - if (p.hdr.getZxid() != zxid) { - // log warning message if there is no matching commit - // old leader send outstanding proposal to observer - LOG.warn( - "Committing 0x{}, but next proposal is 0x{}", - Long.toHexString(zxid), - Long.toHexString(p.hdr.getZxid())); - continue; - } - delayedCommits.remove(); - Request request = new Request(p.hdr.getClientId(), p.hdr.getCxid(), p.hdr.getType(), p.hdr, p.rec, -1); - request.setTxnDigest(p.digest); - ozk.commitRequest(request); - } - } else { - // New server type need to handle in-flight packets - throw new UnsupportedOperationException("Unknown server type"); - } - } - - } - - SyncHelper helper = new SyncHelper(); - QuorumPacket qp = new QuorumPacket(); - readPacket(qp); - synchronized (zk) { - if (qp.getType() == Leader.DIFF) { - LOG.info("Getting a diff from the leader 0x{}", Long.toHexString(qp.getZxid())); - self.setSyncMode(QuorumPeer.SyncMode.DIFF); - if (zk.shouldForceWriteInitialSnapshotAfterLeaderElection()) { - LOG.info("Forcing a snapshot write as part of upgrading from an older Zookeeper. This should only happen while upgrading."); - helper.syncSnapshot(); - } else { - helper.noSnapshot(); - } - } else if (qp.getType() == Leader.SNAP) { - self.setSyncMode(QuorumPeer.SyncMode.SNAP); - LOG.info("Getting a snapshot from leader 0x{}", Long.toHexString(qp.getZxid())); - // The leader is going to dump the database - zk.getZKDatabase().deserializeSnapshot(leaderIs); - // ZOOKEEPER-2819: overwrite config node content extracted - // from leader snapshot with local config, to avoid potential - // inconsistency of config node content during rolling restart. - if (!self.isReconfigEnabled()) { - LOG.debug("Reset config node content from local config after deserialization of snapshot."); - zk.getZKDatabase().initConfigInZKDatabase(self.getQuorumVerifier()); - } - String signature = leaderIs.readString("signature"); - if (!signature.equals("BenWasHere")) { - LOG.error("Missing signature. Got {}", signature); - throw new IOException("Missing signature"); - } - zk.getZKDatabase().setlastProcessedZxid(qp.getZxid()); - - // Immediately persist the latest snapshot when there is txn log gap - helper.syncSnapshot(); - } else if (qp.getType() == Leader.TRUNC) { - // We need to truncate the log to the lastZxid of the leader - self.setSyncMode(QuorumPeer.SyncMode.TRUNC); - LOG.warn("Truncating log to get in sync with the leader 0x{}", Long.toHexString(qp.getZxid())); - boolean truncated = zk.getZKDatabase().truncateLog(qp.getZxid()); - if (!truncated) { - LOG.error("Not able to truncate the log 0x{}", Long.toHexString(qp.getZxid())); - ServiceUtils.requestSystemExit(ExitCode.QUORUM_PACKET_ERROR.getValue()); - } - zk.getZKDatabase().setlastProcessedZxid(qp.getZxid()); - } else { - LOG.error("Got unexpected packet from leader: {}, exiting ... ", LearnerHandler.packetToString(qp)); - ServiceUtils.requestSystemExit(ExitCode.QUORUM_PACKET_ERROR.getValue()); - } - zk.getZKDatabase().initConfigInZKDatabase(self.getQuorumVerifier()); - zk.createSessionTracker(); - - - // we are now going to start getting transactions to apply followed by an UPTODATE - long lastQueued = 0; - TxnLogEntry logEntry; - outerLoop: - while (self.isRunning()) { - readPacket(qp); - switch (qp.getType()) { - case Leader.PROPOSAL: - PacketInFlight pif = new PacketInFlight(); - logEntry = SerializeUtils.deserializeTxn(qp.getData()); - pif.hdr = logEntry.getHeader(); - pif.rec = logEntry.getTxn(); - pif.digest = logEntry.getDigest(); - if (pif.hdr.getZxid() != lastQueued + 1) { - LOG.warn( - "Got zxid 0x{} expected 0x{}", - Long.toHexString(pif.hdr.getZxid()), - Long.toHexString(lastQueued + 1)); - } - lastQueued = pif.hdr.getZxid(); - - if (pif.hdr.getType() == OpCode.reconfig) { - SetDataTxn setDataTxn = (SetDataTxn) pif.rec; - QuorumVerifier qv = self.configFromString(new String(setDataTxn.getData(), UTF_8)); - self.setLastSeenQuorumVerifier(qv, true); - } - helper.propose(pif); - break; - case Leader.COMMIT: - case Leader.COMMITANDACTIVATE: - pif = helper.nextProposal(); - if (pif.hdr.getZxid() != qp.getZxid()) { - LOG.warn( - "Committing 0x{}, but next proposal is 0x{}", - Long.toHexString(qp.getZxid()), - Long.toHexString(pif.hdr.getZxid())); - } else { - if (qp.getType() == Leader.COMMITANDACTIVATE) { - tryReconfig(pif, ByteBuffer.wrap(qp.getData()).getLong(), qp.getZxid()); - } - helper.commit(); - } - break; - case Leader.INFORM: - case Leader.INFORMANDACTIVATE: - PacketInFlight packet = new PacketInFlight(); - if (qp.getType() == Leader.INFORMANDACTIVATE) { - ByteBuffer buffer = ByteBuffer.wrap(qp.getData()); - long suggestedLeaderId = buffer.getLong(); - byte[] remainingData = new byte[buffer.remaining()]; - buffer.get(remainingData); - logEntry = SerializeUtils.deserializeTxn(remainingData); - packet.hdr = logEntry.getHeader(); - packet.rec = logEntry.getTxn(); - packet.digest = logEntry.getDigest(); - tryReconfig(packet, suggestedLeaderId, qp.getZxid()); - } else { - logEntry = SerializeUtils.deserializeTxn(qp.getData()); - packet.rec = logEntry.getTxn(); - packet.hdr = logEntry.getHeader(); - packet.digest = logEntry.getDigest(); - // Log warning message if txn comes out-of-order - if (packet.hdr.getZxid() != lastQueued + 1) { - LOG.warn( - "Got zxid 0x{} expected 0x{}", - Long.toHexString(packet.hdr.getZxid()), - Long.toHexString(lastQueued + 1)); - } - lastQueued = packet.hdr.getZxid(); - } - helper.propose(packet); - helper.commit(); - break; - case Leader.UPTODATE: - LOG.info("Learner received UPTODATE message"); - if (newLeaderQV != null) { - boolean majorChange = self.processReconfig(newLeaderQV, null, null, true); - if (majorChange) { - throw new Exception("changes proposed in reconfig"); - } - } - helper.flushAcks(); - self.setZooKeeperServer(zk); - self.adminServer.setZooKeeperServer(zk); - break outerLoop; - case Leader.NEWLEADER: // Getting NEWLEADER here instead of in discovery - LOG.info("Learner received NEWLEADER message"); - if (qp.getData() != null && qp.getData().length > 1) { - try { - QuorumVerifier qv = self.configFromString(new String(qp.getData(), UTF_8)); - self.setLastSeenQuorumVerifier(qv, true); - newLeaderQV = qv; - } catch (Exception e) { - e.printStackTrace(); - } - } - - helper.writeState(); - writePacket(new QuorumPacket(Leader.ACK, newLeaderZxid, null, null), true); - break; - } - } - } - QuorumPacket ack = new QuorumPacket(Leader.ACK, 0, null, null); - ack.setZxid(ZxidUtils.makeZxid(newEpoch, 0)); - writePacket(ack, true); - zk.startServing(); - /* - * Update the election vote here to ensure that all members of the - * ensemble report the same vote to new servers that start up and - * send leader election notifications to the ensemble. - * - * @see https://issues.apache.org/jira/browse/ZOOKEEPER-1732 - */ - self.updateElectionVote(newEpoch); - - helper.applyDelayedPackets(); - } - - private void tryReconfig(PacketInFlight pif, long newLeader, long zxid) throws Exception { - QuorumVerifier qv = self.configFromString(new String(((SetDataTxn) pif.rec).getData(), UTF_8)); - boolean majorChange = self.processReconfig(qv, newLeader, zxid, true); - if (majorChange) { - throw new Exception("changes proposed in reconfig"); - } - } - - protected void revalidate(QuorumPacket qp) throws IOException { - ByteArrayInputStream bis = new ByteArrayInputStream(qp.getData()); - DataInputStream dis = new DataInputStream(bis); - long sessionId = dis.readLong(); - boolean valid = dis.readBoolean(); - ServerCnxn cnxn = pendingRevalidations.remove(sessionId); - if (cnxn == null) { - LOG.warn("Missing session 0x{} for validation", Long.toHexString(sessionId)); - } else { - zk.finishSessionInit(cnxn, valid); - } - if (LOG.isTraceEnabled()) { - ZooTrace.logTraceMessage( - LOG, - ZooTrace.SESSION_TRACE_MASK, - "Session 0x" + Long.toHexString(sessionId) + " is valid: " + valid); - } - } - - protected void ping(QuorumPacket qp) throws IOException { - // Send back the ping with our session data - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(bos); - Map<Long, Integer> touchTable = zk.getTouchSnapshot(); - for (Entry<Long, Integer> entry : touchTable.entrySet()) { - dos.writeLong(entry.getKey()); - dos.writeInt(entry.getValue()); - } - - QuorumPacket pingReply = new QuorumPacket(qp.getType(), qp.getZxid(), bos.toByteArray(), qp.getAuthinfo()); - writePacket(pingReply, true); - } - - /** - * Shutdown the Peer - */ - public void shutdown() { - self.setZooKeeperServer(null); - self.closeAllConnections(); - self.adminServer.setZooKeeperServer(null); - - if (sender != null) { - sender.shutdown(); - } - - closeSocket(); - // shutdown previous zookeeper - if (zk != null) { - // If we haven't finished SNAP sync, force fully shutdown - // to avoid potential inconsistency - zk.shutdown(self.getSyncMode().equals(QuorumPeer.SyncMode.SNAP)); - } - } - - boolean isRunning() { - return self.isRunning() && zk.isRunning(); - } - - void closeSocket() { - if (sock != null) { - if (sockBeingClosed.compareAndSet(false, true)) { - if (closeSocketAsync) { - final Thread closingThread = new Thread(() -> closeSockSync(), "CloseSocketThread(sid:" + zk.getServerId()); - closingThread.setDaemon(true); - closingThread.start(); - } else { - closeSockSync(); - } - } - } - } - - void closeSockSync() { - try { - long startTime = Time.currentElapsedTime(); - if (sock != null) { - sock.close(); - sock = null; - } - ServerMetrics.getMetrics().SOCKET_CLOSING_TIME.add(Time.currentElapsedTime() - startTime); - } catch (IOException e) { - LOG.warn("Ignoring error closing connection to leader", e); - } - } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java deleted file mode 100644 index 99c4ae16dce..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.quorum; - -import java.io.IOException; -import java.util.Map; -import org.apache.zookeeper.jmx.MBeanRegistry; -import org.apache.zookeeper.server.DataTreeBean; -import org.apache.zookeeper.server.ServerCnxn; -import org.apache.zookeeper.server.SyncRequestProcessor; -import org.apache.zookeeper.server.ZKDatabase; -import org.apache.zookeeper.server.ZooKeeperServerBean; -import org.apache.zookeeper.server.persistence.FileTxnSnapLog; - -/** - * Parent class for all ZooKeeperServers for Learners - */ -public abstract class LearnerZooKeeperServer extends QuorumZooKeeperServer { - - /* - * Request processors - */ - protected CommitProcessor commitProcessor; - protected SyncRequestProcessor syncProcessor; - - public LearnerZooKeeperServer(FileTxnSnapLog logFactory, int tickTime, int minSessionTimeout, int maxSessionTimeout, int listenBacklog, ZKDatabase zkDb, QuorumPeer self) throws IOException { - super(logFactory, tickTime, minSessionTimeout, maxSessionTimeout, listenBacklog, zkDb, self); - } - - /** - * Abstract method to return the learner associated with this server. - * Since the Learner may change under our feet (when QuorumPeer reassigns - * it) we can't simply take a reference here. Instead, we need the - * subclasses to implement this. - */ - public abstract Learner getLearner(); - - /** - * Returns the current state of the session tracker. This is only currently - * used by a Learner to build a ping response packet. - * - */ - protected Map<Long, Integer> getTouchSnapshot() { - if (sessionTracker != null) { - return ((LearnerSessionTracker) sessionTracker).snapshot(); - } - Map<Long, Integer> map = Map.of(); - return map; - } - - /** - * Returns the id of the associated QuorumPeer, which will do for a unique - * id of this server. - */ - @Override - public long getServerId() { - return self.getMyId(); - } - - @Override - public void createSessionTracker() { - sessionTracker = new LearnerSessionTracker( - this, - getZKDatabase().getSessionWithTimeOuts(), - this.tickTime, - self.getMyId(), - self.areLocalSessionsEnabled(), - getZooKeeperServerListener()); - } - - @Override - protected void revalidateSession(ServerCnxn cnxn, long sessionId, int sessionTimeout) throws IOException { - if (upgradeableSessionTracker.isLocalSession(sessionId)) { - super.revalidateSession(cnxn, sessionId, sessionTimeout); - } else { - getLearner().validateSession(cnxn, sessionId, sessionTimeout); - } - } - - @Override - protected void registerJMX() { - // register with JMX - try { - jmxDataTreeBean = new DataTreeBean(getZKDatabase().getDataTree()); - MBeanRegistry.getInstance().register(jmxDataTreeBean, jmxServerBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - jmxDataTreeBean = null; - } - } - - public void registerJMX(ZooKeeperServerBean serverBean, LocalPeerBean localPeerBean) { - // register with JMX - if (self.jmxLeaderElectionBean != null) { - try { - MBeanRegistry.getInstance().unregister(self.jmxLeaderElectionBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - } - self.jmxLeaderElectionBean = null; - } - - try { - jmxServerBean = serverBean; - MBeanRegistry.getInstance().register(serverBean, localPeerBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - jmxServerBean = null; - } - } - - @Override - protected void unregisterJMX() { - // unregister from JMX - try { - if (jmxDataTreeBean != null) { - MBeanRegistry.getInstance().unregister(jmxDataTreeBean); - } - } catch (Exception e) { - LOG.warn("Failed to unregister with JMX", e); - } - jmxDataTreeBean = null; - } - - protected void unregisterJMX(Learner peer) { - // unregister from JMX - try { - if (jmxServerBean != null) { - MBeanRegistry.getInstance().unregister(jmxServerBean); - } - } catch (Exception e) { - LOG.warn("Failed to unregister with JMX", e); - } - jmxServerBean = null; - } - - @Override - public synchronized void shutdown(boolean fullyShutDown) { - if (!canShutdown()) { - LOG.debug("ZooKeeper server is not running, so not proceeding to shutdown!"); - } else { - LOG.info("Shutting down"); - try { - if (syncProcessor != null) { - // Shutting down the syncProcessor here, first, ensures queued transactions here are written to - // permanent storage, which ensures that crash recovery data is consistent with what is used for a - // leader election immediately following shutdown, because of the old leader going down; and also - // that any state on its way to being written is also loaded in the potential call to - // fast-forward-from-edits, in super.shutdown(...), so we avoid getting a DIFF from the new leader - // that contains entries we have already written to our transaction log. - syncProcessor.shutdown(); - } - } catch (Exception e) { - LOG.warn("Ignoring unexpected exception in syncprocessor shutdown", e); - } - } - try { - super.shutdown(fullyShutDown); - } catch (Exception e) { - LOG.warn("Ignoring unexpected exception during shutdown", e); - } - } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java deleted file mode 100644 index 1a44a98e6e7..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.quorum; - -import java.io.IOException; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.function.BiConsumer; -import org.apache.zookeeper.server.FinalRequestProcessor; -import org.apache.zookeeper.server.Request; -import org.apache.zookeeper.server.RequestProcessor; -import org.apache.zookeeper.server.SyncRequestProcessor; -import org.apache.zookeeper.server.ZKDatabase; -import org.apache.zookeeper.server.persistence.FileTxnSnapLog; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A ZooKeeperServer for the Observer node type. Not much is different, but - * we anticipate specializing the request processors in the future. - * - */ -public class ObserverZooKeeperServer extends LearnerZooKeeperServer { - - private static final Logger LOG = LoggerFactory.getLogger(ObserverZooKeeperServer.class); - - /** - * Enable since request processor for writing txnlog to disk and - * take periodic snapshot. Default is ON. - */ - - private boolean syncRequestProcessorEnabled = this.self.getSyncEnabled(); - - /* - * Pending sync requests - */ ConcurrentLinkedQueue<Request> pendingSyncs = new ConcurrentLinkedQueue<>(); - - ObserverZooKeeperServer(FileTxnSnapLog logFactory, QuorumPeer self, ZKDatabase zkDb) throws IOException { - super(logFactory, self.tickTime, self.minSessionTimeout, self.maxSessionTimeout, self.clientPortListenBacklog, zkDb, self); - LOG.info("syncEnabled ={}", syncRequestProcessorEnabled); - } - - public Observer getObserver() { - return self.observer; - } - - @Override - public Learner getLearner() { - return self.observer; - } - - /** - * Unlike a Follower, which sees a full request only during the PROPOSAL - * phase, Observers get all the data required with the INFORM packet. - * This method commits a request that has been unpacked by from an INFORM - * received from the Leader. - * - * @param request - */ - public void commitRequest(Request request) { - if (syncProcessor != null) { - // Write to txnlog and take periodic snapshot - syncProcessor.processRequest(request); - } - commitProcessor.commit(request); - } - - /** - * Set up the request processors for an Observer: - * firstProcesor->commitProcessor->finalProcessor - */ - @Override - protected void setupRequestProcessors() { - // We might consider changing the processor behaviour of - // Observers to, for example, remove the disk sync requirements. - // Currently, they behave almost exactly the same as followers. - RequestProcessor finalProcessor = new FinalRequestProcessor(this); - commitProcessor = new CommitProcessor(finalProcessor, Long.toString(getServerId()), true, getZooKeeperServerListener()); - commitProcessor.start(); - firstProcessor = new ObserverRequestProcessor(this, commitProcessor); - ((ObserverRequestProcessor) firstProcessor).start(); - - /* - * Observer should write to disk, so that the it won't request - * too old txn from the leader which may lead to getting an entire - * snapshot. - * - * However, this may degrade performance as it has to write to disk - * and do periodic snapshot which may double the memory requirements - */ - if (syncRequestProcessorEnabled) { - syncProcessor = new SyncRequestProcessor(this, null); - syncProcessor.start(); - } - } - - /* - * Process a sync request - */ - public synchronized void sync() { - if (pendingSyncs.size() == 0) { - LOG.warn("Not expecting a sync."); - return; - } - - Request r = pendingSyncs.remove(); - commitProcessor.commit(r); - } - - @Override - public String getState() { - return "observer"; - } - - @Override - public void dumpMonitorValues(BiConsumer<String, Object> response) { - super.dumpMonitorValues(response); - response.accept("observer_master_id", getObserver().getLearnerMasterId()); - } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java deleted file mode 100644 index f6fc87d7716..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java +++ /dev/null @@ -1,2711 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.quorum; - -import static org.apache.zookeeper.common.NetUtils.formatInetAddr; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.io.StringReader; -import java.io.StringWriter; -import java.io.Writer; -import java.net.DatagramPacket; -import java.net.DatagramSocket; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import javax.security.sasl.SaslException; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.zookeeper.KeeperException.BadArgumentsException; -import org.apache.zookeeper.common.AtomicFileOutputStream; -import org.apache.zookeeper.common.AtomicFileWritingIdiom; -import org.apache.zookeeper.common.AtomicFileWritingIdiom.WriterStatement; -import org.apache.zookeeper.common.QuorumX509Util; -import org.apache.zookeeper.common.Time; -import org.apache.zookeeper.common.X509Exception; -import org.apache.zookeeper.jmx.MBeanRegistry; -import org.apache.zookeeper.jmx.ZKMBeanInfo; -import org.apache.zookeeper.server.ServerCnxn; -import org.apache.zookeeper.server.ServerCnxnFactory; -import org.apache.zookeeper.server.ServerMetrics; -import org.apache.zookeeper.server.ZKDatabase; -import org.apache.zookeeper.server.ZooKeeperServer; -import org.apache.zookeeper.server.ZooKeeperThread; -import org.apache.zookeeper.server.admin.AdminServer; -import org.apache.zookeeper.server.admin.AdminServer.AdminServerException; -import org.apache.zookeeper.server.admin.AdminServerFactory; -import org.apache.zookeeper.server.persistence.FileTxnSnapLog; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; -import org.apache.zookeeper.server.quorum.auth.NullQuorumAuthLearner; -import org.apache.zookeeper.server.quorum.auth.NullQuorumAuthServer; -import org.apache.zookeeper.server.quorum.auth.QuorumAuth; -import org.apache.zookeeper.server.quorum.auth.QuorumAuthLearner; -import org.apache.zookeeper.server.quorum.auth.QuorumAuthServer; -import org.apache.zookeeper.server.quorum.auth.SaslQuorumAuthLearner; -import org.apache.zookeeper.server.quorum.auth.SaslQuorumAuthServer; -import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; -import org.apache.zookeeper.server.quorum.flexible.QuorumOracleMaj; -import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; -import org.apache.zookeeper.server.util.ConfigUtils; -import org.apache.zookeeper.server.util.JvmPauseMonitor; -import org.apache.zookeeper.server.util.ZxidUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class manages the quorum protocol. There are three states this server - * can be in: - * <ol> - * <li>Leader election - each server will elect a leader (proposing itself as a - * leader initially).</li> - * <li>Follower - the server will synchronize with the leader and replicate any - * transactions.</li> - * <li>Leader - the server will process requests and forward them to followers. - * A majority of followers must log the request before it can be accepted. - * </ol> - * - * This class will setup a datagram socket that will always respond with its - * view of the current leader. The response will take the form of: - * - * <pre> - * int xid; - * - * long myid; - * - * long leader_id; - * - * long leader_zxid; - * </pre> - * - * The request for the current leader will consist solely of an xid: int xid; - */ -public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider { - - private static final Logger LOG = LoggerFactory.getLogger(QuorumPeer.class); - - public static final String CONFIG_KEY_KERBEROS_CANONICALIZE_HOST_NAMES = "zookeeper.kerberos.canonicalizeHostNames"; - public static final String CONFIG_DEFAULT_KERBEROS_CANONICALIZE_HOST_NAMES = "false"; - - private QuorumBean jmxQuorumBean; - LocalPeerBean jmxLocalPeerBean; - private Map<Long, RemotePeerBean> jmxRemotePeerBean; - LeaderElectionBean jmxLeaderElectionBean; - - // The QuorumCnxManager is held through an AtomicReference to ensure cross-thread visibility - // of updates; see the implementation comment at setLastSeenQuorumVerifier(). - private AtomicReference<QuorumCnxManager> qcmRef = new AtomicReference<>(); - - QuorumAuthServer authServer; - QuorumAuthLearner authLearner; - - /** - * ZKDatabase is a top level member of quorumpeer - * which will be used in all the zookeeperservers - * instantiated later. Also, it is created once on - * bootup and only thrown away in case of a truncate - * message from the leader - */ - private ZKDatabase zkDb; - - private JvmPauseMonitor jvmPauseMonitor; - - private final AtomicBoolean suspended = new AtomicBoolean(false); - - public static final class AddressTuple { - - public final MultipleAddresses quorumAddr; - public final MultipleAddresses electionAddr; - public final InetSocketAddress clientAddr; - - public AddressTuple(MultipleAddresses quorumAddr, MultipleAddresses electionAddr, InetSocketAddress clientAddr) { - this.quorumAddr = quorumAddr; - this.electionAddr = electionAddr; - this.clientAddr = clientAddr; - } - - } - - private int observerMasterPort; - - public int getObserverMasterPort() { - return observerMasterPort; - } - - public void setObserverMasterPort(int observerMasterPort) { - this.observerMasterPort = observerMasterPort; - } - - public static final String CONFIG_KEY_MULTI_ADDRESS_ENABLED = "zookeeper.multiAddress.enabled"; - public static final String CONFIG_DEFAULT_MULTI_ADDRESS_ENABLED = "false"; - - private boolean multiAddressEnabled = true; - public boolean isMultiAddressEnabled() { - return multiAddressEnabled; - } - - public void setMultiAddressEnabled(boolean multiAddressEnabled) { - this.multiAddressEnabled = multiAddressEnabled; - LOG.info("multiAddress.enabled set to {}", multiAddressEnabled); - } - - public static final String CONFIG_KEY_MULTI_ADDRESS_REACHABILITY_CHECK_TIMEOUT_MS = "zookeeper.multiAddress.reachabilityCheckTimeoutMs"; - - private int multiAddressReachabilityCheckTimeoutMs = (int) MultipleAddresses.DEFAULT_TIMEOUT.toMillis(); - public int getMultiAddressReachabilityCheckTimeoutMs() { - return multiAddressReachabilityCheckTimeoutMs; - } - - public void setMultiAddressReachabilityCheckTimeoutMs(int multiAddressReachabilityCheckTimeoutMs) { - this.multiAddressReachabilityCheckTimeoutMs = multiAddressReachabilityCheckTimeoutMs; - LOG.info("multiAddress.reachabilityCheckTimeoutMs set to {}", multiAddressReachabilityCheckTimeoutMs); - } - - public static final String CONFIG_KEY_MULTI_ADDRESS_REACHABILITY_CHECK_ENABLED = "zookeeper.multiAddress.reachabilityCheckEnabled"; - - private boolean multiAddressReachabilityCheckEnabled = true; - - public boolean isMultiAddressReachabilityCheckEnabled() { - return multiAddressReachabilityCheckEnabled; - } - - public void setMultiAddressReachabilityCheckEnabled(boolean multiAddressReachabilityCheckEnabled) { - this.multiAddressReachabilityCheckEnabled = multiAddressReachabilityCheckEnabled; - LOG.info("multiAddress.reachabilityCheckEnabled set to {}", multiAddressReachabilityCheckEnabled); - } - - public static class QuorumServer { - - public MultipleAddresses addr = new MultipleAddresses(); - - public MultipleAddresses electionAddr = new MultipleAddresses(); - - public InetSocketAddress clientAddr = null; - - public long id; - - public String hostname; - - public LearnerType type = LearnerType.PARTICIPANT; - - public boolean isClientAddrFromStatic = false; - - private List<InetSocketAddress> myAddrs; - - public QuorumServer(long id, InetSocketAddress addr, InetSocketAddress electionAddr, InetSocketAddress clientAddr) { - this(id, addr, electionAddr, clientAddr, LearnerType.PARTICIPANT); - } - - public QuorumServer(long id, InetSocketAddress addr, InetSocketAddress electionAddr) { - this(id, addr, electionAddr, null, LearnerType.PARTICIPANT); - } - - // VisibleForTesting - public QuorumServer(long id, InetSocketAddress addr) { - this(id, addr, null, null, LearnerType.PARTICIPANT); - } - - public long getId() { - return id; - } - - /** - * Performs a DNS lookup for server address and election address. - * - * If the DNS lookup fails, this.addr and electionAddr remain - * unmodified. - */ - public void recreateSocketAddresses() { - if (this.addr.isEmpty()) { - LOG.warn("Server address has not been initialized"); - return; - } - if (this.electionAddr.isEmpty()) { - LOG.warn("Election address has not been initialized"); - return; - } - this.addr.recreateSocketAddresses(); - this.electionAddr.recreateSocketAddresses(); - } - - private LearnerType getType(String s) throws ConfigException { - switch (s.trim().toLowerCase()) { - case "observer": - return LearnerType.OBSERVER; - case "participant": - return LearnerType.PARTICIPANT; - default: - throw new ConfigException("Unrecognised peertype: " + s); - } - } - - public QuorumServer(long sid, String addressStr) throws ConfigException { - this(sid, addressStr, QuorumServer::getInetAddress); - } - - QuorumServer(long sid, String addressStr, Function<InetSocketAddress, InetAddress> getInetAddress) throws ConfigException { - this.id = sid; - initializeWithAddressString(addressStr, getInetAddress); - } - - public QuorumServer(long id, InetSocketAddress addr, InetSocketAddress electionAddr, LearnerType type) { - this(id, addr, electionAddr, null, type); - } - - public QuorumServer(long id, InetSocketAddress addr, InetSocketAddress electionAddr, InetSocketAddress clientAddr, LearnerType type) { - this.id = id; - if (addr != null) { - this.addr.addAddress(addr); - } - if (electionAddr != null) { - this.electionAddr.addAddress(electionAddr); - } - this.type = type; - this.clientAddr = clientAddr; - - setMyAddrs(); - } - - private static final String wrongFormat = - " does not have the form server_config or server_config;client_config" - + " where server_config is the pipe separated list of host:port:port or host:port:port:type" - + " and client_config is port or host:port"; - - private void initializeWithAddressString(String addressStr, Function<InetSocketAddress, InetAddress> getInetAddress) throws ConfigException { - LearnerType newType = null; - String[] serverClientParts = addressStr.split(";"); - String[] serverAddresses = serverClientParts[0].split("\\|"); - - if (serverClientParts.length == 2) { - String[] clientParts = ConfigUtils.getHostAndPort(serverClientParts[1]); - if (clientParts.length > 2) { - throw new ConfigException(addressStr + wrongFormat); - } - - // is client_config a host:port or just a port - String clientHostName = (clientParts.length == 2) ? clientParts[0] : "0.0.0.0"; - try { - clientAddr = new InetSocketAddress(clientHostName, Integer.parseInt(clientParts[clientParts.length - 1])); - } catch (NumberFormatException e) { - throw new ConfigException("Address unresolved: " + hostname + ":" + clientParts[clientParts.length - 1]); - } - } - - boolean multiAddressEnabled = Boolean.parseBoolean( - System.getProperty(QuorumPeer.CONFIG_KEY_MULTI_ADDRESS_ENABLED, QuorumPeer.CONFIG_DEFAULT_MULTI_ADDRESS_ENABLED)); - if (!multiAddressEnabled && serverAddresses.length > 1) { - throw new ConfigException("Multiple address feature is disabled, but multiple addresses were specified for sid " + this.id); - } - - boolean canonicalize = Boolean.parseBoolean( - System.getProperty( - CONFIG_KEY_KERBEROS_CANONICALIZE_HOST_NAMES, - CONFIG_DEFAULT_KERBEROS_CANONICALIZE_HOST_NAMES)); - - for (String serverAddress : serverAddresses) { - String serverParts[] = ConfigUtils.getHostAndPort(serverAddress); - if ((serverClientParts.length > 2) || (serverParts.length < 3) - || (serverParts.length > 4)) { - throw new ConfigException(addressStr + wrongFormat); - } - - String serverHostName = serverParts[0]; - - // server_config should be either host:port:port or host:port:port:type - InetSocketAddress tempAddress; - InetSocketAddress tempElectionAddress; - try { - tempAddress = new InetSocketAddress(serverHostName, Integer.parseInt(serverParts[1])); - addr.addAddress(tempAddress); - } catch (NumberFormatException e) { - throw new ConfigException("Address unresolved: " + serverHostName + ":" + serverParts[1]); - } - try { - tempElectionAddress = new InetSocketAddress(serverHostName, Integer.parseInt(serverParts[2])); - electionAddr.addAddress(tempElectionAddress); - } catch (NumberFormatException e) { - throw new ConfigException("Address unresolved: " + serverHostName + ":" + serverParts[2]); - } - - if (tempAddress.getPort() == tempElectionAddress.getPort()) { - throw new ConfigException("Client and election port must be different! Please update the " - + "configuration file on server." + this.id); - } - - if (canonicalize) { - InetAddress ia = getInetAddress.apply(tempAddress); - if (ia == null) { - throw new ConfigException("Unable to canonicalize address " + serverHostName + " because it's not resolvable"); - } - - String canonicalHostName = ia.getCanonicalHostName(); - - if (!canonicalHostName.equals(serverHostName) - // Avoid using literal IP address when - // security check fails - && !canonicalHostName.equals(ia.getHostAddress())) { - LOG.info("Host name for quorum server {} " - + "canonicalized from {} to {}", - this.id, serverHostName, canonicalHostName); - serverHostName = canonicalHostName; - } - } - - if (serverParts.length == 4) { - LearnerType tempType = getType(serverParts[3]); - if (newType == null) { - newType = tempType; - } - - if (newType != tempType) { - throw new ConfigException("Multiple addresses should have similar roles: " + type + " vs " + tempType); - } - } - - this.hostname = serverHostName; - } - - if (newType != null) { - type = newType; - } - - setMyAddrs(); - } - - private static InetAddress getInetAddress(InetSocketAddress addr) { - return addr.getAddress(); - } - - private void setMyAddrs() { - this.myAddrs = new ArrayList<>(); - this.myAddrs.addAll(this.addr.getAllAddresses()); - this.myAddrs.add(this.clientAddr); - this.myAddrs.addAll(this.electionAddr.getAllAddresses()); - this.myAddrs = excludedSpecialAddresses(this.myAddrs); - } - - public static String delimitedHostString(InetSocketAddress addr) { - String host = addr.getHostString(); - if (host.contains(":")) { - return "[" + host + "]"; - } else { - return host; - } - } - - public String toString() { - StringWriter sw = new StringWriter(); - - List<InetSocketAddress> addrList = new LinkedList<>(addr.getAllAddresses()); - List<InetSocketAddress> electionAddrList = new LinkedList<>(electionAddr.getAllAddresses()); - - if (addrList.size() > 0 && electionAddrList.size() > 0) { - addrList.sort(Comparator.comparing(InetSocketAddress::getHostString)); - electionAddrList.sort(Comparator.comparing(InetSocketAddress::getHostString)); - sw.append(IntStream.range(0, addrList.size()).mapToObj(i -> String.format("%s:%d:%d", - delimitedHostString(addrList.get(i)), addrList.get(i).getPort(), electionAddrList.get(i).getPort())) - .collect(Collectors.joining("|"))); - } - - if (type == LearnerType.OBSERVER) { - sw.append(":observer"); - } else if (type == LearnerType.PARTICIPANT) { - sw.append(":participant"); - } - - if (clientAddr != null && !isClientAddrFromStatic) { - sw.append(";"); - sw.append(delimitedHostString(clientAddr)); - sw.append(":"); - sw.append(String.valueOf(clientAddr.getPort())); - } - - return sw.toString(); - } - - public int hashCode() { - assert false : "hashCode not designed"; - return 42; // any arbitrary constant will do - } - - private boolean checkAddressesEqual(InetSocketAddress addr1, InetSocketAddress addr2) { - return (addr1 != null || addr2 == null) - && (addr1 == null || addr2 != null) - && (addr1 == null || addr2 == null || addr1.equals(addr2)); - } - - public boolean equals(Object o) { - if (!(o instanceof QuorumServer)) { - return false; - } - QuorumServer qs = (QuorumServer) o; - if ((qs.id != id) || (qs.type != type)) { - return false; - } - if (!addr.equals(qs.addr)) { - return false; - } - if (!electionAddr.equals(qs.electionAddr)) { - return false; - } - return checkAddressesEqual(clientAddr, qs.clientAddr); - } - - public void checkAddressDuplicate(QuorumServer s) throws BadArgumentsException { - List<InetSocketAddress> otherAddrs = new ArrayList<>(s.addr.getAllAddresses()); - otherAddrs.add(s.clientAddr); - otherAddrs.addAll(s.electionAddr.getAllAddresses()); - otherAddrs = excludedSpecialAddresses(otherAddrs); - - for (InetSocketAddress my : this.myAddrs) { - - for (InetSocketAddress other : otherAddrs) { - if (my.equals(other)) { - String error = String.format("%s of server.%d conflicts %s of server.%d", my, this.id, other, s.id); - throw new BadArgumentsException(error); - } - } - } - } - - private List<InetSocketAddress> excludedSpecialAddresses(List<InetSocketAddress> addrs) { - List<InetSocketAddress> included = new ArrayList<>(); - - for (InetSocketAddress addr : addrs) { - if (addr == null) { - continue; - } - InetAddress inetaddr = addr.getAddress(); - - if (inetaddr == null || inetaddr.isAnyLocalAddress() // wildCard addresses (0.0.0.0 or [::]) - || inetaddr.isLoopbackAddress()) { // loopback address(localhost/127.0.0.1) - continue; - } - included.add(addr); - } - return included; - } - - } - - public enum ServerState { - LOOKING, - FOLLOWING, - LEADING, - OBSERVING - } - - /** - * (Used for monitoring) shows the current phase of - * Zab protocol that peer is running. - */ - public enum ZabState { - ELECTION, - DISCOVERY, - SYNCHRONIZATION, - BROADCAST - } - - /** - * (Used for monitoring) When peer is in synchronization phase, this shows - * which synchronization mechanism is being used - */ - public enum SyncMode { - NONE, - DIFF, - SNAP, - TRUNC - } - - /* - * A peer can either be participating, which implies that it is willing to - * both vote in instances of consensus and to elect or become a Leader, or - * it may be observing in which case it isn't. - * - * We need this distinction to decide which ServerState to move to when - * conditions change (e.g. which state to become after LOOKING). - */ - public enum LearnerType { - PARTICIPANT, - OBSERVER - } - - /* - * To enable observers to have no identifier, we need a generic identifier - * at least for QuorumCnxManager. We use the following constant to as the - * value of such a generic identifier. - */ - - static final long OBSERVER_ID = Long.MAX_VALUE; - - /* - * Record leader election time - */ - public long start_fle, end_fle; // fle = fast leader election - public static final String FLE_TIME_UNIT = "MS"; - private long unavailableStartTime; - - /* - * Default value of peer is participant - */ - private LearnerType learnerType = LearnerType.PARTICIPANT; - - public LearnerType getLearnerType() { - return learnerType; - } - - /** - * Sets the LearnerType - */ - public void setLearnerType(LearnerType p) { - learnerType = p; - } - - protected synchronized void setConfigFileName(String s) { - configFilename = s; - } - - private String configFilename = null; - - public int getQuorumSize() { - return getVotingView().size(); - } - - public void setJvmPauseMonitor(JvmPauseMonitor jvmPauseMonitor) { - this.jvmPauseMonitor = jvmPauseMonitor; - } - - /** - * QuorumVerifier implementation; default (majority). - */ - - //last committed quorum verifier - private QuorumVerifier quorumVerifier; - - //last proposed quorum verifier - private QuorumVerifier lastSeenQuorumVerifier = null; - - // Lock object that guard access to quorumVerifier and lastSeenQuorumVerifier. - final Object QV_LOCK = new Object(); - - /** - * My id - */ - private long myid; - - /** - * get the id of this quorum peer. - */ - public long getMyId() { - return myid; - } - - // VisibleForTesting - void setId(long id) { - this.myid = id; - } - - private boolean sslQuorum; - private boolean shouldUsePortUnification; - - public boolean isSslQuorum() { - return sslQuorum; - } - - public boolean shouldUsePortUnification() { - return shouldUsePortUnification; - } - - private final QuorumX509Util x509Util; - - QuorumX509Util getX509Util() { - return x509Util; - } - - /** - * This is who I think the leader currently is. - */ - private volatile Vote currentVote; - - public synchronized Vote getCurrentVote() { - return currentVote; - } - - public synchronized void setCurrentVote(Vote v) { - currentVote = v; - } - - private volatile boolean running = true; - - private String initialConfig; - - /** - * The number of milliseconds of each tick - */ - protected int tickTime; - - /** - * Whether learners in this quorum should create new sessions as local. - * False by default to preserve existing behavior. - */ - protected boolean localSessionsEnabled = false; - - /** - * Whether learners in this quorum should upgrade local sessions to - * global. Only matters if local sessions are enabled. - */ - protected boolean localSessionsUpgradingEnabled = true; - - /** - * Minimum number of milliseconds to allow for session timeout. - * A value of -1 indicates unset, use default. - */ - protected int minSessionTimeout = -1; - - /** - * Maximum number of milliseconds to allow for session timeout. - * A value of -1 indicates unset, use default. - */ - protected int maxSessionTimeout = -1; - - /** - * The ZooKeeper server's socket backlog length. The number of connections - * that will be queued to be read before new connections are dropped. A - * value of one indicates the default backlog will be used. - */ - protected int clientPortListenBacklog = -1; - - /** - * The number of ticks that the initial synchronization phase can take - */ - protected volatile int initLimit; - - /** - * The number of ticks that can pass between sending a request and getting - * an acknowledgment - */ - protected volatile int syncLimit; - - /** - * The number of ticks that can pass before retrying to connect to learner master - */ - protected volatile int connectToLearnerMasterLimit; - - /** - * Enables/Disables sync request processor. This option is enabled - * by default and is to be used with observers. - */ - protected boolean syncEnabled = true; - - /** - * The current tick - */ - protected AtomicInteger tick = new AtomicInteger(); - - /** - * Whether or not to listen on all IPs for the two quorum ports - * (broadcast and fast leader election). - */ - protected boolean quorumListenOnAllIPs = false; - - /** - * Keeps time taken for leader election in milliseconds. Sets the value to - * this variable only after the completion of leader election. - */ - private long electionTimeTaken = -1; - - /** - * Enable/Disables quorum authentication using sasl. Defaulting to false. - */ - protected boolean quorumSaslEnableAuth; - - /** - * If this is false, quorum peer server will accept another quorum peer client - * connection even if the authentication did not succeed. This can be used while - * upgrading ZooKeeper server. Defaulting to false (required). - */ - protected boolean quorumServerSaslAuthRequired; - - /** - * If this is false, quorum peer learner will talk to quorum peer server - * without authentication. This can be used while upgrading ZooKeeper - * server. Defaulting to false (required). - */ - protected boolean quorumLearnerSaslAuthRequired; - - /** - * Kerberos quorum service principal. Defaulting to 'zkquorum/localhost'. - */ - protected String quorumServicePrincipal; - - /** - * Quorum learner login context name in jaas-conf file to read the kerberos - * security details. Defaulting to 'QuorumLearner'. - */ - protected String quorumLearnerLoginContext; - - /** - * Quorum server login context name in jaas-conf file to read the kerberos - * security details. Defaulting to 'QuorumServer'. - */ - protected String quorumServerLoginContext; - - // TODO: need to tune the default value of thread size - private static final int QUORUM_CNXN_THREADS_SIZE_DEFAULT_VALUE = 20; - /** - * The maximum number of threads to allow in the connectionExecutors thread - * pool which will be used to initiate quorum server connections. - */ - protected int quorumCnxnThreadsSize = QUORUM_CNXN_THREADS_SIZE_DEFAULT_VALUE; - - public static final String QUORUM_CNXN_TIMEOUT_MS = "zookeeper.quorumCnxnTimeoutMs"; - private static int quorumCnxnTimeoutMs; - - static { - quorumCnxnTimeoutMs = Integer.getInteger(QUORUM_CNXN_TIMEOUT_MS, -1); - LOG.info("{}={}", QUORUM_CNXN_TIMEOUT_MS, quorumCnxnTimeoutMs); - } - - /** - * @deprecated As of release 3.4.0, this class has been deprecated, since - * it is used with one of the udp-based versions of leader election, which - * we are also deprecating. - * - * This class simply responds to requests for the current leader of this - * node. - * <p> - * The request contains just an xid generated by the requestor. - * <p> - * The response has the xid, the id of this server, the id of the leader, - * and the zxid of the leader. - * - * - */ - @Deprecated - class ResponderThread extends ZooKeeperThread { - - ResponderThread() { - super("ResponderThread"); - } - - volatile boolean running = true; - - @Override - public void run() { - try { - byte[] b = new byte[36]; - ByteBuffer responseBuffer = ByteBuffer.wrap(b); - DatagramPacket packet = new DatagramPacket(b, b.length); - while (running) { - udpSocket.receive(packet); - if (packet.getLength() != 4) { - LOG.warn("Got more than just an xid! Len = {}", packet.getLength()); - } else { - responseBuffer.clear(); - responseBuffer.getInt(); // Skip the xid - responseBuffer.putLong(myid); - Vote current = getCurrentVote(); - switch (getPeerState()) { - case LOOKING: - responseBuffer.putLong(current.getId()); - responseBuffer.putLong(current.getZxid()); - break; - case LEADING: - responseBuffer.putLong(myid); - try { - long proposed; - synchronized (leader) { - proposed = leader.lastProposed; - } - responseBuffer.putLong(proposed); - } catch (NullPointerException npe) { - // This can happen in state transitions, - // just ignore the request - } - break; - case FOLLOWING: - responseBuffer.putLong(current.getId()); - try { - responseBuffer.putLong(follower.getZxid()); - } catch (NullPointerException npe) { - // This can happen in state transitions, - // just ignore the request - } - break; - case OBSERVING: - // Do nothing, Observers keep themselves to - // themselves. - break; - } - packet.setData(b); - udpSocket.send(packet); - } - packet.setLength(b.length); - } - } catch (RuntimeException e) { - LOG.warn("Unexpected runtime exception in ResponderThread", e); - } catch (IOException e) { - LOG.warn("Unexpected IO exception in ResponderThread", e); - } finally { - LOG.warn("QuorumPeer responder thread exited"); - } - } - - } - - private ServerState state = ServerState.LOOKING; - - private AtomicReference<ZabState> zabState = new AtomicReference<>(ZabState.ELECTION); - private AtomicReference<SyncMode> syncMode = new AtomicReference<>(SyncMode.NONE); - private AtomicReference<String> leaderAddress = new AtomicReference<>(""); - private AtomicLong leaderId = new AtomicLong(-1); - - private boolean reconfigFlag = false; // indicates that a reconfig just committed - - public synchronized void setPeerState(ServerState newState) { - state = newState; - if (newState == ServerState.LOOKING) { - setLeaderAddressAndId(null, -1); - setZabState(ZabState.ELECTION); - } else { - LOG.info("Peer state changed: {}", getDetailedPeerState()); - } - } - - public void setZabState(ZabState zabState) { - if ((zabState == ZabState.BROADCAST) && (unavailableStartTime != 0)) { - long unavailableTime = Time.currentElapsedTime() - unavailableStartTime; - ServerMetrics.getMetrics().UNAVAILABLE_TIME.add(unavailableTime); - if (getPeerState() == ServerState.LEADING) { - ServerMetrics.getMetrics().LEADER_UNAVAILABLE_TIME.add(unavailableTime); - } - unavailableStartTime = 0; - } - this.zabState.set(zabState); - LOG.info("Peer state changed: {}", getDetailedPeerState()); - } - - public void setSyncMode(SyncMode syncMode) { - this.syncMode.set(syncMode); - LOG.info("Peer state changed: {}", getDetailedPeerState()); - } - - public ZabState getZabState() { - return zabState.get(); - } - - public SyncMode getSyncMode() { - return syncMode.get(); - } - - public void setLeaderAddressAndId(MultipleAddresses addr, long newId) { - if (addr != null) { - leaderAddress.set(String.join("|", addr.getAllHostStrings())); - } else { - leaderAddress.set(null); - } - leaderId.set(newId); - } - - public String getLeaderAddress() { - return leaderAddress.get(); - } - - public long getLeaderId() { - return leaderId.get(); - } - - public String getDetailedPeerState() { - final StringBuilder sb = new StringBuilder(getPeerState().toString().toLowerCase()); - final ZabState zabState = getZabState(); - if (!ZabState.ELECTION.equals(zabState)) { - sb.append(" - ").append(zabState.toString().toLowerCase()); - } - final SyncMode syncMode = getSyncMode(); - if (!SyncMode.NONE.equals(syncMode)) { - sb.append(" - ").append(syncMode.toString().toLowerCase()); - } - return sb.toString(); - } - - public synchronized void reconfigFlagSet() { - reconfigFlag = true; - } - public synchronized void reconfigFlagClear() { - reconfigFlag = false; - } - public synchronized boolean isReconfigStateChange() { - return reconfigFlag; - } - public synchronized ServerState getPeerState() { - return state; - } - - DatagramSocket udpSocket; - - private final AtomicReference<AddressTuple> myAddrs = new AtomicReference<>(); - - /** - * Resolves hostname for a given server ID. - * - * This method resolves hostname for a given server ID in both quorumVerifer - * and lastSeenQuorumVerifier. If the server ID matches the local server ID, - * it also updates myAddrs. - */ - public void recreateSocketAddresses(long id) { - QuorumVerifier qv = getQuorumVerifier(); - if (qv != null) { - QuorumServer qs = qv.getAllMembers().get(id); - if (qs != null) { - qs.recreateSocketAddresses(); - if (id == getMyId()) { - setAddrs(qs.addr, qs.electionAddr, qs.clientAddr); - } - } - } - qv = getLastSeenQuorumVerifier(); - if (qv != null) { - QuorumServer qs = qv.getAllMembers().get(id); - if (qs != null) { - qs.recreateSocketAddresses(); - } - } - } - - private AddressTuple getAddrs() { - AddressTuple addrs = myAddrs.get(); - if (addrs != null) { - return addrs; - } - try { - synchronized (QV_LOCK) { - addrs = myAddrs.get(); - while (addrs == null) { - QV_LOCK.wait(); - addrs = myAddrs.get(); - } - return addrs; - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - } - - public MultipleAddresses getQuorumAddress() { - return getAddrs().quorumAddr; - } - - public MultipleAddresses getElectionAddress() { - return getAddrs().electionAddr; - } - - public InetSocketAddress getClientAddress() { - final AddressTuple addrs = myAddrs.get(); - return (addrs == null) ? null : addrs.clientAddr; - } - - private void setAddrs(MultipleAddresses quorumAddr, MultipleAddresses electionAddr, InetSocketAddress clientAddr) { - synchronized (QV_LOCK) { - myAddrs.set(new AddressTuple(quorumAddr, electionAddr, clientAddr)); - QV_LOCK.notifyAll(); - } - } - - private int electionType; - - Election electionAlg; - - ServerCnxnFactory cnxnFactory; - ServerCnxnFactory secureCnxnFactory; - - private FileTxnSnapLog logFactory = null; - - private final QuorumStats quorumStats; - - AdminServer adminServer; - - private final boolean reconfigEnabled; - - public static QuorumPeer testingQuorumPeer() throws SaslException { - return new QuorumPeer(); - } - - public QuorumPeer() throws SaslException { - super("QuorumPeer"); - quorumStats = new QuorumStats(this); - jmxRemotePeerBean = new HashMap<>(); - adminServer = AdminServerFactory.createAdminServer(); - x509Util = createX509Util(); - initialize(); - reconfigEnabled = QuorumPeerConfig.isReconfigEnabled(); - } - - // VisibleForTesting - QuorumX509Util createX509Util() { - return new QuorumX509Util(); - } - - /** - * For backward compatibility purposes, we instantiate QuorumMaj by default. - */ - - public QuorumPeer(Map<Long, QuorumServer> quorumPeers, File dataDir, File dataLogDir, int electionType, long myid, int tickTime, int initLimit, int syncLimit, int connectToLearnerMasterLimit, ServerCnxnFactory cnxnFactory) throws IOException { - this(quorumPeers, dataDir, dataLogDir, electionType, myid, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit, false, cnxnFactory, new QuorumMaj(quorumPeers)); - } - - public QuorumPeer(Map<Long, QuorumServer> quorumPeers, File dataDir, File dataLogDir, int electionType, long myid, int tickTime, int initLimit, int syncLimit, int connectToLearnerMasterLimit, boolean quorumListenOnAllIPs, ServerCnxnFactory cnxnFactory, QuorumVerifier quorumConfig) throws IOException { - this(); - this.cnxnFactory = cnxnFactory; - this.electionType = electionType; - this.myid = myid; - this.tickTime = tickTime; - this.initLimit = initLimit; - this.syncLimit = syncLimit; - this.connectToLearnerMasterLimit = connectToLearnerMasterLimit; - this.quorumListenOnAllIPs = quorumListenOnAllIPs; - this.logFactory = new FileTxnSnapLog(dataLogDir, dataDir); - this.zkDb = new ZKDatabase(this.logFactory); - if (quorumConfig == null) { - quorumConfig = new QuorumMaj(quorumPeers); - } - setQuorumVerifier(quorumConfig, false); - adminServer = AdminServerFactory.createAdminServer(); - } - - public void initialize() throws SaslException { - // init quorum auth server & learner - if (isQuorumSaslAuthEnabled()) { - Set<String> authzHosts = new HashSet<>(); - for (QuorumServer qs : getView().values()) { - authzHosts.add(qs.hostname); - } - authServer = new SaslQuorumAuthServer(isQuorumServerSaslAuthRequired(), quorumServerLoginContext, authzHosts); - authLearner = new SaslQuorumAuthLearner(isQuorumLearnerSaslAuthRequired(), quorumServicePrincipal, quorumLearnerLoginContext); - } else { - authServer = new NullQuorumAuthServer(); - authLearner = new NullQuorumAuthLearner(); - } - } - - QuorumStats quorumStats() { - return quorumStats; - } - - @Override - public synchronized void start() { - if (!getView().containsKey(myid)) { - throw new RuntimeException("My id " + myid + " not in the peer list"); - } - loadDataBase(); - startServerCnxnFactory(); - try { - adminServer.start(); - } catch (AdminServerException e) { - LOG.warn("Problem starting AdminServer", e); - } - startLeaderElection(); - startJvmPauseMonitor(); - super.start(); - } - - private void loadDataBase() { - try { - zkDb.loadDataBase(); - - // load the epochs - long lastProcessedZxid = zkDb.getDataTree().lastProcessedZxid; - long epochOfZxid = ZxidUtils.getEpochFromZxid(lastProcessedZxid); - try { - currentEpoch = readLongFromFile(CURRENT_EPOCH_FILENAME); - } catch (FileNotFoundException e) { - // pick a reasonable epoch number - // this should only happen once when moving to a - // new code version - currentEpoch = epochOfZxid; - LOG.info( - "{} not found! Creating with a reasonable default of {}. " - + "This should only happen when you are upgrading your installation", - CURRENT_EPOCH_FILENAME, - currentEpoch); - writeLongToFile(CURRENT_EPOCH_FILENAME, currentEpoch); - } - if (epochOfZxid > currentEpoch) { - // acceptedEpoch.tmp file in snapshot directory - File currentTmp = new File(getTxnFactory().getSnapDir(), - CURRENT_EPOCH_FILENAME + AtomicFileOutputStream.TMP_EXTENSION); - if (currentTmp.exists()) { - long epochOfTmp = readLongFromFile(currentTmp.getName()); - LOG.info("{} found. Setting current epoch to {}.", currentTmp, epochOfTmp); - setCurrentEpoch(epochOfTmp); - } else { - throw new IOException( - "The current epoch, " + ZxidUtils.zxidToString(currentEpoch) - + ", is older than the last zxid, " + lastProcessedZxid); - } - } - try { - acceptedEpoch = readLongFromFile(ACCEPTED_EPOCH_FILENAME); - } catch (FileNotFoundException e) { - // pick a reasonable epoch number - // this should only happen once when moving to a - // new code version - acceptedEpoch = epochOfZxid; - LOG.info( - "{} not found! Creating with a reasonable default of {}. " - + "This should only happen when you are upgrading your installation", - ACCEPTED_EPOCH_FILENAME, - acceptedEpoch); - writeLongToFile(ACCEPTED_EPOCH_FILENAME, acceptedEpoch); - } - if (acceptedEpoch < currentEpoch) { - throw new IOException("The accepted epoch, " - + ZxidUtils.zxidToString(acceptedEpoch) - + " is less than the current epoch, " - + ZxidUtils.zxidToString(currentEpoch)); - } - } catch (IOException ie) { - LOG.error("Unable to load database on disk", ie); - throw new RuntimeException("Unable to run quorum server ", ie); - } - } - - ResponderThread responder; - - public synchronized void stopLeaderElection() { - responder.running = false; - responder.interrupt(); - } - public synchronized void startLeaderElection() { - try { - if (getPeerState() == ServerState.LOOKING) { - currentVote = new Vote(myid, getLastLoggedZxid(), getCurrentEpoch()); - } - } catch (IOException e) { - RuntimeException re = new RuntimeException(e.getMessage()); - re.setStackTrace(e.getStackTrace()); - throw re; - } - - this.electionAlg = createElectionAlgorithm(electionType); - } - - private void startJvmPauseMonitor() { - if (this.jvmPauseMonitor != null) { - this.jvmPauseMonitor.serviceStart(); - } - } - - /** - * Count the number of nodes in the map that could be followers. - * @param peers - * @return The number of followers in the map - */ - protected static int countParticipants(Map<Long, QuorumServer> peers) { - int count = 0; - for (QuorumServer q : peers.values()) { - if (q.type == LearnerType.PARTICIPANT) { - count++; - } - } - return count; - } - - /** - * This constructor is only used by the existing unit test code. - * It defaults to FileLogProvider persistence provider. - */ - public QuorumPeer(Map<Long, QuorumServer> quorumPeers, File snapDir, File logDir, int clientPort, int electionAlg, long myid, int tickTime, int initLimit, int syncLimit, int connectToLearnerMasterLimit) throws IOException { - this( - quorumPeers, - snapDir, - logDir, - electionAlg, - myid, - tickTime, - initLimit, - syncLimit, - connectToLearnerMasterLimit, - false, - ServerCnxnFactory.createFactory(getClientAddress(quorumPeers, myid, clientPort), -1), - new QuorumMaj(quorumPeers)); - } - - public QuorumPeer(Map<Long, QuorumServer> quorumPeers, File snapDir, File logDir, int clientPort, int electionAlg, long myid, int tickTime, int initLimit, int syncLimit, int connectToLearnerMasterLimit, String oraclePath) throws IOException { - this( - quorumPeers, - snapDir, - logDir, - electionAlg, - myid, - tickTime, - initLimit, - syncLimit, - connectToLearnerMasterLimit, - false, - ServerCnxnFactory.createFactory(getClientAddress(quorumPeers, myid, clientPort), -1), - new QuorumOracleMaj(quorumPeers, oraclePath)); - } - - /** - * This constructor is only used by the existing unit test code. - * It defaults to FileLogProvider persistence provider. - */ - public QuorumPeer(Map<Long, QuorumServer> quorumPeers, File snapDir, File logDir, int clientPort, int electionAlg, long myid, int tickTime, int initLimit, int syncLimit, int connectToLearnerMasterLimit, QuorumVerifier quorumConfig) throws IOException { - this( - quorumPeers, - snapDir, - logDir, - electionAlg, - myid, - tickTime, - initLimit, - syncLimit, - connectToLearnerMasterLimit, - false, - ServerCnxnFactory.createFactory(getClientAddress(quorumPeers, myid, clientPort), -1), - quorumConfig); - } - - private static InetSocketAddress getClientAddress(Map<Long, QuorumServer> quorumPeers, long myid, int clientPort) throws IOException { - QuorumServer quorumServer = quorumPeers.get(myid); - if (null == quorumServer) { - throw new IOException("No QuorumServer correspoding to myid " + myid); - } - if (null == quorumServer.clientAddr) { - return new InetSocketAddress(clientPort); - } - if (quorumServer.clientAddr.getPort() != clientPort) { - throw new IOException("QuorumServer port " - + quorumServer.clientAddr.getPort() - + " does not match with given port " - + clientPort); - } - return quorumServer.clientAddr; - } - - /** - * returns the highest zxid that this host has seen - * - * @return the highest zxid for this host - */ - public long getLastLoggedZxid() { - if (!zkDb.isInitialized()) { - loadDataBase(); - } - return zkDb.getDataTreeLastProcessedZxid(); - } - - public Follower follower; - public Leader leader; - public Observer observer; - - protected Follower makeFollower(FileTxnSnapLog logFactory) throws IOException { - return new Follower(this, new FollowerZooKeeperServer(logFactory, this, this.zkDb)); - } - - protected Leader makeLeader(FileTxnSnapLog logFactory) throws IOException, X509Exception { - return new Leader(this, new LeaderZooKeeperServer(logFactory, this, this.zkDb)); - } - - protected Observer makeObserver(FileTxnSnapLog logFactory) throws IOException { - return new Observer(this, new ObserverZooKeeperServer(logFactory, this, this.zkDb)); - } - - @SuppressWarnings("deprecation") - protected Election createElectionAlgorithm(int electionAlgorithm) { - Election le = null; - - //TODO: use a factory rather than a switch - switch (electionAlgorithm) { - case 1: - throw new UnsupportedOperationException("Election Algorithm 1 is not supported."); - case 2: - throw new UnsupportedOperationException("Election Algorithm 2 is not supported."); - case 3: - QuorumCnxManager qcm = createCnxnManager(); - QuorumCnxManager oldQcm = qcmRef.getAndSet(qcm); - if (oldQcm != null) { - LOG.warn("Clobbering already-set QuorumCnxManager (restarting leader election?)"); - oldQcm.halt(); - } - QuorumCnxManager.Listener listener = qcm.listener; - if (listener != null) { - listener.start(); - FastLeaderElection fle = new FastLeaderElection(this, qcm); - fle.start(); - le = fle; - } else { - LOG.error("Null listener when initializing cnx manager"); - } - break; - default: - assert false; - } - return le; - } - - @SuppressWarnings("deprecation") - protected Election makeLEStrategy() { - LOG.debug("Initializing leader election protocol..."); - return electionAlg; - } - - protected synchronized void setLeader(Leader newLeader) { - leader = newLeader; - } - - protected synchronized void setFollower(Follower newFollower) { - follower = newFollower; - } - - protected synchronized void setObserver(Observer newObserver) { - observer = newObserver; - } - - public synchronized ZooKeeperServer getActiveServer() { - if (leader != null) { - return leader.zk; - } else if (follower != null) { - return follower.zk; - } else if (observer != null) { - return observer.zk; - } - return null; - } - - boolean shuttingDownLE = false; - - public void setSuspended(boolean suspended) { - this.suspended.set(suspended); - } - private void checkSuspended() { - try { - while (suspended.get()) { - Thread.sleep(10); - } - } catch (InterruptedException err) { - Thread.currentThread().interrupt(); - } - } - - @Override - public void run() { - updateThreadName(); - - LOG.debug("Starting quorum peer"); - try { - jmxQuorumBean = new QuorumBean(this); - MBeanRegistry.getInstance().register(jmxQuorumBean, null); - for (QuorumServer s : getView().values()) { - ZKMBeanInfo p; - if (getMyId() == s.id) { - p = jmxLocalPeerBean = new LocalPeerBean(this); - try { - MBeanRegistry.getInstance().register(p, jmxQuorumBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - jmxLocalPeerBean = null; - } - } else { - RemotePeerBean rBean = new RemotePeerBean(this, s); - try { - MBeanRegistry.getInstance().register(rBean, jmxQuorumBean); - jmxRemotePeerBean.put(s.id, rBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - } - } - } - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - jmxQuorumBean = null; - } - - try { - /* - * Main loop - */ - while (running) { - if (unavailableStartTime == 0) { - unavailableStartTime = Time.currentElapsedTime(); - } - - switch (getPeerState()) { - case LOOKING: - LOG.info("LOOKING"); - ServerMetrics.getMetrics().LOOKING_COUNT.add(1); - - if (Boolean.getBoolean("readonlymode.enabled")) { - LOG.info("Attempting to start ReadOnlyZooKeeperServer"); - - // Create read-only server but don't start it immediately - final ReadOnlyZooKeeperServer roZk = new ReadOnlyZooKeeperServer(logFactory, this, this.zkDb); - - // Instead of starting roZk immediately, wait some grace - // period before we decide we're partitioned. - // - // Thread is used here because otherwise it would require - // changes in each of election strategy classes which is - // unnecessary code coupling. - Thread roZkMgr = new Thread() { - public void run() { - try { - // lower-bound grace period to 2 secs - sleep(Math.max(2000, tickTime)); - if (ServerState.LOOKING.equals(getPeerState())) { - roZk.startup(); - } - } catch (InterruptedException e) { - LOG.info("Interrupted while attempting to start ReadOnlyZooKeeperServer, not started"); - } catch (Exception e) { - LOG.error("FAILED to start ReadOnlyZooKeeperServer", e); - } - } - }; - try { - roZkMgr.start(); - reconfigFlagClear(); - if (shuttingDownLE) { - shuttingDownLE = false; - startLeaderElection(); - } - setCurrentVote(makeLEStrategy().lookForLeader()); - checkSuspended(); - } catch (Exception e) { - LOG.warn("Unexpected exception", e); - setPeerState(ServerState.LOOKING); - } finally { - // If the thread is in the the grace period, interrupt - // to come out of waiting. - roZkMgr.interrupt(); - roZk.shutdown(); - } - } else { - try { - reconfigFlagClear(); - if (shuttingDownLE) { - shuttingDownLE = false; - startLeaderElection(); - } - setCurrentVote(makeLEStrategy().lookForLeader()); - } catch (Exception e) { - LOG.warn("Unexpected exception", e); - setPeerState(ServerState.LOOKING); - } - } - break; - case OBSERVING: - try { - LOG.info("OBSERVING"); - setObserver(makeObserver(logFactory)); - observer.observeLeader(); - } catch (Exception e) { - LOG.warn("Unexpected exception", e); - } finally { - observer.shutdown(); - setObserver(null); - updateServerState(); - - // Add delay jitter before we switch to LOOKING - // state to reduce the load of ObserverMaster - if (isRunning()) { - Observer.waitForObserverElectionDelay(); - } - } - break; - case FOLLOWING: - try { - LOG.info("FOLLOWING"); - setFollower(makeFollower(logFactory)); - follower.followLeader(); - } catch (Exception e) { - LOG.warn("Unexpected exception", e); - } finally { - follower.shutdown(); - setFollower(null); - updateServerState(); - } - break; - case LEADING: - LOG.info("LEADING"); - try { - setLeader(makeLeader(logFactory)); - leader.lead(); - setLeader(null); - } catch (Exception e) { - LOG.warn("Unexpected exception", e); - } finally { - if (leader != null) { - leader.shutdown("Forcing shutdown"); - setLeader(null); - } - updateServerState(); - } - break; - } - } - } finally { - LOG.warn("QuorumPeer main thread exited"); - MBeanRegistry instance = MBeanRegistry.getInstance(); - instance.unregister(jmxQuorumBean); - instance.unregister(jmxLocalPeerBean); - - for (RemotePeerBean remotePeerBean : jmxRemotePeerBean.values()) { - instance.unregister(remotePeerBean); - } - - jmxQuorumBean = null; - jmxLocalPeerBean = null; - jmxRemotePeerBean = null; - } - } - - private synchronized void updateServerState() { - if (!reconfigFlag) { - setPeerState(ServerState.LOOKING); - LOG.warn("PeerState set to LOOKING"); - return; - } - - if (getMyId() == getCurrentVote().getId()) { - setPeerState(ServerState.LEADING); - LOG.debug("PeerState set to LEADING"); - } else if (getLearnerType() == LearnerType.PARTICIPANT) { - setPeerState(ServerState.FOLLOWING); - LOG.debug("PeerState set to FOLLOWING"); - } else if (getLearnerType() == LearnerType.OBSERVER) { - setPeerState(ServerState.OBSERVING); - LOG.debug("PeerState set to OBSERVER"); - } else { // currently shouldn't happen since there are only 2 learner types - setPeerState(ServerState.LOOKING); - LOG.debug("Should not be here"); - } - reconfigFlag = false; - } - - public void shutdown() { - running = false; - x509Util.close(); - if (leader != null) { - leader.shutdown("quorum Peer shutdown"); - } - if (follower != null) { - follower.shutdown(); - } - shutdownServerCnxnFactory(); - if (udpSocket != null) { - udpSocket.close(); - } - if (jvmPauseMonitor != null) { - jvmPauseMonitor.serviceStop(); - } - - try { - adminServer.shutdown(); - } catch (AdminServerException e) { - LOG.warn("Problem stopping AdminServer", e); - } - - if (getElectionAlg() != null) { - this.interrupt(); - getElectionAlg().shutdown(); - } - try { - zkDb.close(); - } catch (IOException ie) { - LOG.warn("Error closing logs ", ie); - } - } - - /** - * A 'view' is a node's current opinion of the membership of the entire - * ensemble. - */ - public Map<Long, QuorumPeer.QuorumServer> getView() { - return Collections.unmodifiableMap(getQuorumVerifier().getAllMembers()); - } - - /** - * Observers are not contained in this view, only nodes with - * PeerType=PARTICIPANT. - */ - public Map<Long, QuorumPeer.QuorumServer> getVotingView() { - return getQuorumVerifier().getVotingMembers(); - } - - /** - * Returns only observers, no followers. - */ - public Map<Long, QuorumPeer.QuorumServer> getObservingView() { - return getQuorumVerifier().getObservingMembers(); - } - - public synchronized Set<Long> getCurrentAndNextConfigVoters() { - Set<Long> voterIds = new HashSet<>(getQuorumVerifier().getVotingMembers().keySet()); - if (getLastSeenQuorumVerifier() != null) { - voterIds.addAll(getLastSeenQuorumVerifier().getVotingMembers().keySet()); - } - return voterIds; - } - - /** - * Check if a node is in the current view. With static membership, the - * result of this check will never change; only when dynamic membership - * is introduced will this be more useful. - */ - public boolean viewContains(Long sid) { - return this.getView().containsKey(sid); - } - - /** - * Only used by QuorumStats at the moment - */ - public String[] getQuorumPeers() { - List<String> l = new ArrayList<>(); - synchronized (this) { - if (leader != null) { - for (LearnerHandler fh : leader.getLearners()) { - if (fh.getSocket() != null) { - String s = formatInetAddr((InetSocketAddress) fh.getSocket().getRemoteSocketAddress()); - if (leader.isLearnerSynced(fh)) { - s += "*"; - } - l.add(s); - } - } - } else if (follower != null) { - l.add(formatInetAddr((InetSocketAddress) follower.sock.getRemoteSocketAddress())); - } - } - return l.toArray(new String[0]); - } - - public String getServerState() { - switch (getPeerState()) { - case LOOKING: - return QuorumStats.Provider.LOOKING_STATE; - case LEADING: - return QuorumStats.Provider.LEADING_STATE; - case FOLLOWING: - return QuorumStats.Provider.FOLLOWING_STATE; - case OBSERVING: - return QuorumStats.Provider.OBSERVING_STATE; - } - return QuorumStats.Provider.UNKNOWN_STATE; - } - - /** - * set the id of this quorum peer. - */ - public void setMyid(long myid) { - this.myid = myid; - } - - public void setInitialConfig(String initialConfig) { - this.initialConfig = initialConfig; - } - - public String getInitialConfig() { - return initialConfig; - } - - /** - * Get the number of milliseconds of each tick - */ - public int getTickTime() { - return tickTime; - } - - /** - * Set the number of milliseconds of each tick - */ - public void setTickTime(int tickTime) { - LOG.info("tickTime set to {}", tickTime); - this.tickTime = tickTime; - } - - /** Maximum number of connections allowed from particular host (ip) */ - public int getMaxClientCnxnsPerHost() { - if (cnxnFactory != null) { - return cnxnFactory.getMaxClientCnxnsPerHost(); - } - if (secureCnxnFactory != null) { - return secureCnxnFactory.getMaxClientCnxnsPerHost(); - } - return -1; - } - - /** Whether local sessions are enabled */ - public boolean areLocalSessionsEnabled() { - return localSessionsEnabled; - } - - /** Whether to enable local sessions */ - public void enableLocalSessions(boolean flag) { - LOG.info("Local sessions {}", (flag ? "enabled" : "disabled")); - localSessionsEnabled = flag; - } - - /** Whether local sessions are allowed to upgrade to global sessions */ - public boolean isLocalSessionsUpgradingEnabled() { - return localSessionsUpgradingEnabled; - } - - /** Whether to allow local sessions to upgrade to global sessions */ - public void enableLocalSessionsUpgrading(boolean flag) { - LOG.info("Local session upgrading {}", (flag ? "enabled" : "disabled")); - localSessionsUpgradingEnabled = flag; - } - - /** minimum session timeout in milliseconds */ - public int getMinSessionTimeout() { - return minSessionTimeout; - } - - /** minimum session timeout in milliseconds */ - public void setMinSessionTimeout(int min) { - LOG.info("minSessionTimeout set to {}", min); - this.minSessionTimeout = min; - } - - /** maximum session timeout in milliseconds */ - public int getMaxSessionTimeout() { - return maxSessionTimeout; - } - - /** maximum session timeout in milliseconds */ - public void setMaxSessionTimeout(int max) { - LOG.info("maxSessionTimeout set to {}", max); - this.maxSessionTimeout = max; - } - - /** The server socket's listen backlog length */ - public int getClientPortListenBacklog() { - return this.clientPortListenBacklog; - } - - /** Sets the server socket's listen backlog length. */ - public void setClientPortListenBacklog(int backlog) { - this.clientPortListenBacklog = backlog; - } - - /** - * Get the number of ticks that the initial synchronization phase can take - */ - public int getInitLimit() { - return initLimit; - } - - /** - * Set the number of ticks that the initial synchronization phase can take - */ - public void setInitLimit(int initLimit) { - LOG.info("initLimit set to {}", initLimit); - this.initLimit = initLimit; - } - - /** - * Get the current tick - */ - public int getTick() { - return tick.get(); - } - - public QuorumVerifier configFromString(String s) throws IOException, ConfigException { - Properties props = new Properties(); - props.load(new StringReader(s)); - return QuorumPeerConfig.parseDynamicConfig(props, electionType, false, false, getQuorumVerifier().getOraclePath()); - } - - /** - * Return QuorumVerifier object for the last committed configuration. - */ - public QuorumVerifier getQuorumVerifier() { - synchronized (QV_LOCK) { - return quorumVerifier; - } - } - - /** - * Return QuorumVerifier object for the last proposed configuration. - */ - public QuorumVerifier getLastSeenQuorumVerifier() { - synchronized (QV_LOCK) { - return lastSeenQuorumVerifier; - } - } - - public synchronized void restartLeaderElection(QuorumVerifier qvOLD, QuorumVerifier qvNEW) { - if (qvOLD == null || !qvOLD.equals(qvNEW)) { - LOG.warn("Restarting Leader Election"); - getElectionAlg().shutdown(); - shuttingDownLE = false; - startLeaderElection(); - } - } - - public String getNextDynamicConfigFilename() { - if (configFilename == null) { - LOG.warn("configFilename is null! This should only happen in tests."); - return null; - } - return configFilename + QuorumPeerConfig.nextDynamicConfigFileSuffix; - } - - // On entry to this method, qcm must be non-null and the locks on both qcm and QV_LOCK - // must be held. We don't want quorumVerifier/lastSeenQuorumVerifier to change out from - // under us, so we have to hold QV_LOCK; and since the call to qcm.connectOne() will take - // the lock on qcm (and take QV_LOCK again inside that), the caller needs to have taken - // qcm outside QV_LOCK to avoid a deadlock against other callers of qcm.connectOne(). - private void connectNewPeers(QuorumCnxManager qcm) { - if (quorumVerifier != null && lastSeenQuorumVerifier != null) { - Map<Long, QuorumServer> committedView = quorumVerifier.getAllMembers(); - for (Entry<Long, QuorumServer> e : lastSeenQuorumVerifier.getAllMembers().entrySet()) { - if (e.getKey() != getMyId() && !committedView.containsKey(e.getKey())) { - qcm.connectOne(e.getKey()); - } - } - } - } - - public void setLastSeenQuorumVerifier(QuorumVerifier qv, boolean writeToDisk) { - if (!isReconfigEnabled()) { - LOG.info("Dynamic reconfig is disabled, we don't store the last seen config."); - return; - } - - // If qcm is non-null, we may call qcm.connectOne(), which will take the lock on qcm - // and then take QV_LOCK. Take the locks in the same order to ensure that we don't - // deadlock against other callers of connectOne(). If qcmRef gets set in another - // thread while we're inside the synchronized block, that does no harm; if we didn't - // take a lock on qcm (because it was null when we sampled it), we won't call - // connectOne() on it. (Use of an AtomicReference is enough to guarantee visibility - // of updates that provably happen in another thread before entering this method.) - QuorumCnxManager qcm = qcmRef.get(); - Object outerLockObject = (qcm != null) ? qcm : QV_LOCK; - synchronized (outerLockObject) { - synchronized (QV_LOCK) { - if (lastSeenQuorumVerifier != null && lastSeenQuorumVerifier.getVersion() > qv.getVersion()) { - LOG.error("setLastSeenQuorumVerifier called with stale config " - + qv.getVersion() - + ". Current version: " - + quorumVerifier.getVersion()); - } - // assuming that a version uniquely identifies a configuration, so if - // version is the same, nothing to do here. - if (lastSeenQuorumVerifier != null && lastSeenQuorumVerifier.getVersion() == qv.getVersion()) { - return; - } - lastSeenQuorumVerifier = qv; - if (qcm != null) { - connectNewPeers(qcm); - } - - if (writeToDisk) { - try { - String fileName = getNextDynamicConfigFilename(); - if (fileName != null) { - QuorumPeerConfig.writeDynamicConfig(fileName, qv, true); - } - } catch (IOException e) { - LOG.error("Error writing next dynamic config file to disk", e); - } - } - } - } - } - - public QuorumVerifier setQuorumVerifier(QuorumVerifier qv, boolean writeToDisk) { - synchronized (QV_LOCK) { - if ((quorumVerifier != null) && (quorumVerifier.getVersion() >= qv.getVersion())) { - // this is normal. For example - server found out about new config through FastLeaderElection gossiping - // and then got the same config in UPTODATE message so its already known - LOG.debug( - "{} setQuorumVerifier called with known or old config {}. Current version: {}", - getMyId(), - qv.getVersion(), - quorumVerifier.getVersion()); - return quorumVerifier; - } - QuorumVerifier prevQV = quorumVerifier; - quorumVerifier = qv; - if (lastSeenQuorumVerifier == null || (qv.getVersion() > lastSeenQuorumVerifier.getVersion())) { - lastSeenQuorumVerifier = qv; - } - - if (writeToDisk) { - // some tests initialize QuorumPeer without a static config file - if (configFilename != null) { - try { - String dynamicConfigFilename = makeDynamicConfigFilename(qv.getVersion()); - QuorumPeerConfig.writeDynamicConfig(dynamicConfigFilename, qv, false); - QuorumPeerConfig.editStaticConfig(configFilename, dynamicConfigFilename, needEraseClientInfoFromStaticConfig()); - } catch (IOException e) { - LOG.error("Error closing file", e); - } - } else { - LOG.info("writeToDisk == true but configFilename == null"); - } - } - - if (qv.getVersion() == lastSeenQuorumVerifier.getVersion()) { - QuorumPeerConfig.deleteFile(getNextDynamicConfigFilename()); - } - QuorumServer qs = qv.getAllMembers().get(getMyId()); - if (qs != null) { - setAddrs(qs.addr, qs.electionAddr, qs.clientAddr); - } - updateObserverMasterList(); - return prevQV; - } - } - - private String makeDynamicConfigFilename(long version) { - return configFilename + ".dynamic." + Long.toHexString(version); - } - - private boolean needEraseClientInfoFromStaticConfig() { - QuorumServer server = quorumVerifier.getAllMembers().get(getMyId()); - return (server != null && server.clientAddr != null && !server.isClientAddrFromStatic); - } - - /** - * Get an instance of LeaderElection - */ - public Election getElectionAlg() { - return electionAlg; - } - - /** - * Get the synclimit - */ - public int getSyncLimit() { - return syncLimit; - } - - /** - * Set the synclimit - */ - public void setSyncLimit(int syncLimit) { - LOG.info("syncLimit set to {}", syncLimit); - this.syncLimit = syncLimit; - } - - /** - * Get the connectToLearnerMasterLimit - */ - public int getConnectToLearnerMasterLimit() { - return connectToLearnerMasterLimit; - } - - /** - * Set the connectToLearnerMasterLimit - */ - public void setConnectToLearnerMasterLimit(int connectToLearnerMasterLimit) { - LOG.info("connectToLearnerMasterLimit set to {}", connectToLearnerMasterLimit); - this.connectToLearnerMasterLimit = connectToLearnerMasterLimit; - } - - /** - * The syncEnabled can also be set via a system property. - */ - public static final String SYNC_ENABLED = "zookeeper.observer.syncEnabled"; - - /** - * Return syncEnabled. - */ - public boolean getSyncEnabled() { - if (System.getProperty(SYNC_ENABLED) != null) { - LOG.info("{}={}", SYNC_ENABLED, Boolean.getBoolean(SYNC_ENABLED)); - return Boolean.getBoolean(SYNC_ENABLED); - } else { - return syncEnabled; - } - } - - /** - * Set syncEnabled. - * - * @param syncEnabled - */ - public void setSyncEnabled(boolean syncEnabled) { - this.syncEnabled = syncEnabled; - } - - /** - * Gets the election type - */ - public int getElectionType() { - return electionType; - } - - /** - * Sets the election type - */ - public void setElectionType(int electionType) { - this.electionType = electionType; - } - - public boolean getQuorumListenOnAllIPs() { - return quorumListenOnAllIPs; - } - - public void setQuorumListenOnAllIPs(boolean quorumListenOnAllIPs) { - this.quorumListenOnAllIPs = quorumListenOnAllIPs; - } - - public void setCnxnFactory(ServerCnxnFactory cnxnFactory) { - this.cnxnFactory = cnxnFactory; - } - - public void setSecureCnxnFactory(ServerCnxnFactory secureCnxnFactory) { - this.secureCnxnFactory = secureCnxnFactory; - } - - public void setSslQuorum(boolean sslQuorum) { - if (sslQuorum) { - LOG.info("Using TLS encrypted quorum communication"); - } else { - LOG.info("Using insecure (non-TLS) quorum communication"); - } - this.sslQuorum = sslQuorum; - } - - public void setUsePortUnification(boolean shouldUsePortUnification) { - LOG.info("Port unification {}", shouldUsePortUnification ? "enabled" : "disabled"); - this.shouldUsePortUnification = shouldUsePortUnification; - } - - private void startServerCnxnFactory() { - if (cnxnFactory != null) { - cnxnFactory.start(); - } - if (secureCnxnFactory != null) { - secureCnxnFactory.start(); - } - } - - private void shutdownServerCnxnFactory() { - if (cnxnFactory != null) { - cnxnFactory.shutdown(); - } - if (secureCnxnFactory != null) { - secureCnxnFactory.shutdown(); - } - } - - // Leader and learner will control the zookeeper server and pass it into QuorumPeer. - public void setZooKeeperServer(ZooKeeperServer zks) { - if (cnxnFactory != null) { - cnxnFactory.setZooKeeperServer(zks); - } - if (secureCnxnFactory != null) { - secureCnxnFactory.setZooKeeperServer(zks); - } - } - - public void closeAllConnections() { - if (cnxnFactory != null) { - cnxnFactory.closeAll(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN); - } - if (secureCnxnFactory != null) { - secureCnxnFactory.closeAll(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN); - } - } - - public int getClientPort() { - if (cnxnFactory != null) { - return cnxnFactory.getLocalPort(); - } - return -1; - } - - public int getSecureClientPort() { - if (secureCnxnFactory != null) { - return secureCnxnFactory.getLocalPort(); - } - return -1; - } - - public void setTxnFactory(FileTxnSnapLog factory) { - this.logFactory = factory; - } - - public FileTxnSnapLog getTxnFactory() { - return this.logFactory; - } - - /** - * set zk database for this node - * @param database - */ - public void setZKDatabase(ZKDatabase database) { - this.zkDb = database; - } - - protected ZKDatabase getZkDb() { - return zkDb; - } - - public synchronized void initConfigInZKDatabase() { - if (zkDb != null) { - zkDb.initConfigInZKDatabase(getQuorumVerifier()); - } - } - - public boolean isRunning() { - return running; - } - - /** - * get reference to QuorumCnxManager - */ - public QuorumCnxManager getQuorumCnxManager() { - return qcmRef.get(); - } - private long readLongFromFile(String name) throws IOException { - File file = new File(logFactory.getSnapDir(), name); - BufferedReader br = new BufferedReader(new FileReader(file)); - String line = ""; - try { - line = br.readLine(); - return Long.parseLong(line); - } catch (NumberFormatException e) { - throw new IOException("Found " + line + " in " + file); - } finally { - br.close(); - } - } - - private long acceptedEpoch = -1; - private long currentEpoch = -1; - - public static final String CURRENT_EPOCH_FILENAME = "currentEpoch"; - - public static final String ACCEPTED_EPOCH_FILENAME = "acceptedEpoch"; - - /** - * Write a long value to disk atomically. Either succeeds or an exception - * is thrown. - * @param name file name to write the long to - * @param value the long value to write to the named file - * @throws IOException if the file cannot be written atomically - */ - // visibleForTest - void writeLongToFile(String name, final long value) throws IOException { - File file = new File(logFactory.getSnapDir(), name); - new AtomicFileWritingIdiom(file, new WriterStatement() { - @Override - public void write(Writer bw) throws IOException { - bw.write(Long.toString(value)); - } - }); - } - - public long getCurrentEpoch() throws IOException { - if (currentEpoch == -1) { - currentEpoch = readLongFromFile(CURRENT_EPOCH_FILENAME); - } - return currentEpoch; - } - - public long getAcceptedEpoch() throws IOException { - if (acceptedEpoch == -1) { - acceptedEpoch = readLongFromFile(ACCEPTED_EPOCH_FILENAME); - } - return acceptedEpoch; - } - - public void setCurrentEpoch(long e) throws IOException { - writeLongToFile(CURRENT_EPOCH_FILENAME, e); - currentEpoch = e; - } - - public void setAcceptedEpoch(long e) throws IOException { - writeLongToFile(ACCEPTED_EPOCH_FILENAME, e); - acceptedEpoch = e; - } - - public boolean processReconfig(QuorumVerifier qv, Long suggestedLeaderId, Long zxid, boolean restartLE) { - if (!isReconfigEnabled()) { - LOG.debug("Reconfig feature is disabled, skip reconfig processing."); - return false; - } - - InetSocketAddress oldClientAddr = getClientAddress(); - - // update last committed quorum verifier, write the new config to disk - // and restart leader election if config changed. - QuorumVerifier prevQV = setQuorumVerifier(qv, true); - - // There is no log record for the initial config, thus after syncing - // with leader - // /zookeeper/config is empty! it is also possible that last committed - // config is propagated during leader election - // without the propagation the corresponding log records. - // so we should explicitly do this (this is not necessary when we're - // already a Follower/Observer, only - // for Learner): - initConfigInZKDatabase(); - - if (prevQV.getVersion() < qv.getVersion() && !prevQV.equals(qv)) { - Map<Long, QuorumServer> newMembers = qv.getAllMembers(); - updateRemotePeerMXBeans(newMembers); - if (restartLE) { - restartLeaderElection(prevQV, qv); - } - - QuorumServer myNewQS = newMembers.get(getMyId()); - if (myNewQS != null && myNewQS.clientAddr != null && !myNewQS.clientAddr.equals(oldClientAddr)) { - cnxnFactory.reconfigure(myNewQS.clientAddr); - updateThreadName(); - } - - boolean roleChange = updateLearnerType(qv); - boolean leaderChange = false; - if (suggestedLeaderId != null) { - // zxid should be non-null too - leaderChange = updateVote(suggestedLeaderId, zxid); - } else { - long currentLeaderId = getCurrentVote().getId(); - QuorumServer myleaderInCurQV = prevQV.getVotingMembers().get(currentLeaderId); - QuorumServer myleaderInNewQV = qv.getVotingMembers().get(currentLeaderId); - leaderChange = (myleaderInCurQV == null - || myleaderInCurQV.addr == null - || myleaderInNewQV == null - || !myleaderInCurQV.addr.equals(myleaderInNewQV.addr)); - // we don't have a designated leader - need to go into leader - // election - reconfigFlagClear(); - } - - return roleChange || leaderChange; - } - return false; - - } - - private void updateRemotePeerMXBeans(Map<Long, QuorumServer> newMembers) { - Set<Long> existingMembers = new HashSet<>(newMembers.keySet()); - existingMembers.retainAll(jmxRemotePeerBean.keySet()); - for (Long id : existingMembers) { - RemotePeerBean rBean = jmxRemotePeerBean.get(id); - rBean.setQuorumServer(newMembers.get(id)); - } - - Set<Long> joiningMembers = new HashSet<>(newMembers.keySet()); - joiningMembers.removeAll(jmxRemotePeerBean.keySet()); - joiningMembers.remove(getMyId()); // remove self as it is local bean - for (Long id : joiningMembers) { - QuorumServer qs = newMembers.get(id); - RemotePeerBean rBean = new RemotePeerBean(this, qs); - try { - MBeanRegistry.getInstance().register(rBean, jmxQuorumBean); - jmxRemotePeerBean.put(qs.id, rBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - } - } - - Set<Long> leavingMembers = new HashSet<>(jmxRemotePeerBean.keySet()); - leavingMembers.removeAll(newMembers.keySet()); - for (Long id : leavingMembers) { - RemotePeerBean rBean = jmxRemotePeerBean.remove(id); - try { - MBeanRegistry.getInstance().unregister(rBean); - } catch (Exception e) { - LOG.warn("Failed to unregister with JMX", e); - } - } - } - - private ArrayList<QuorumServer> observerMasters = new ArrayList<>(); - private void updateObserverMasterList() { - if (observerMasterPort <= 0) { - return; // observer masters not enabled - } - observerMasters.clear(); - StringBuilder sb = new StringBuilder(); - for (QuorumServer server : quorumVerifier.getVotingMembers().values()) { - InetAddress address = server.addr.getReachableOrOne().getAddress(); - InetSocketAddress addr = new InetSocketAddress(address, observerMasterPort); - observerMasters.add(new QuorumServer(server.id, addr)); - sb.append(addr).append(","); - } - LOG.info("Updated learner master list to be {}", sb.toString()); - Collections.shuffle(observerMasters); - // Reset the internal index of the observerMaster when - // the observerMaster List is refreshed - nextObserverMaster = 0; - } - - private boolean useObserverMasters() { - return getLearnerType() == LearnerType.OBSERVER && observerMasters.size() > 0; - } - - private int nextObserverMaster = 0; - private QuorumServer nextObserverMaster() { - if (nextObserverMaster >= observerMasters.size()) { - nextObserverMaster = 0; - // Add a reconnect delay only after the observer - // has exhausted trying to connect to all the masters - // from the observerMasterList - if (isRunning()) { - Observer.waitForReconnectDelay(); - } - } - return observerMasters.get(nextObserverMaster++); - } - - QuorumServer findLearnerMaster(QuorumServer leader) { - if (useObserverMasters()) { - return nextObserverMaster(); - } else { - // Add delay jitter to reduce the load on the leader - if (isRunning()) { - Observer.waitForReconnectDelay(); - } - return leader; - } - } - - /** - * Vet a given learner master's information. - * Allows specification by server id, ip only, or ip and port - */ - QuorumServer validateLearnerMaster(String desiredMaster) { - if (useObserverMasters()) { - Long sid; - try { - sid = Long.parseLong(desiredMaster); - } catch (NumberFormatException e) { - sid = null; - } - for (QuorumServer server : observerMasters) { - if (sid == null) { - for (InetSocketAddress address : server.addr.getAllAddresses()) { - String serverAddr = address.getAddress().getHostAddress() + ':' + address.getPort(); - if (serverAddr.startsWith(desiredMaster)) { - return server; - } - } - } else { - if (sid.equals(server.id)) { - return server; - } - } - } - if (sid == null) { - LOG.info("could not find learner master address={}", desiredMaster); - } else { - LOG.warn("could not find learner master sid={}", sid); - } - } else { - LOG.info("cannot validate request, observer masters not enabled"); - } - return null; - } - - private boolean updateLearnerType(QuorumVerifier newQV) { - //check if I'm an observer in new config - if (newQV.getObservingMembers().containsKey(getMyId())) { - if (getLearnerType() != LearnerType.OBSERVER) { - setLearnerType(LearnerType.OBSERVER); - LOG.info("Becoming an observer"); - reconfigFlagSet(); - return true; - } else { - return false; - } - } else if (newQV.getVotingMembers().containsKey(getMyId())) { - if (getLearnerType() != LearnerType.PARTICIPANT) { - setLearnerType(LearnerType.PARTICIPANT); - LOG.info("Becoming a voting participant"); - reconfigFlagSet(); - return true; - } else { - return false; - } - } - // I'm not in the view - if (getLearnerType() != LearnerType.PARTICIPANT) { - setLearnerType(LearnerType.PARTICIPANT); - LOG.info("Becoming a non-voting participant"); - reconfigFlagSet(); - return true; - } - return false; - } - - private boolean updateVote(long designatedLeader, long zxid) { - Vote currentVote = getCurrentVote(); - if (currentVote != null && designatedLeader != currentVote.getId()) { - setCurrentVote(new Vote(designatedLeader, zxid)); - reconfigFlagSet(); - LOG.warn("Suggested leader: {}", designatedLeader); - return true; - } - return false; - } - - /** - * Updates leader election info to avoid inconsistencies when - * a new server tries to join the ensemble. - * - * Here is the inconsistency scenario we try to solve by updating the peer - * epoch after following leader: - * - * Let's say we have an ensemble with 3 servers z1, z2 and z3. - * - * 1. z1, z2 were following z3 with peerEpoch to be 0xb8, the new epoch is - * 0xb9, aka current accepted epoch on disk. - * 2. z2 get restarted, which will use 0xb9 as it's peer epoch when loading - * the current accept epoch from disk. - * 3. z2 received notification from z1 and z3, which is following z3 with - * epoch 0xb8, so it started following z3 again with peer epoch 0xb8. - * 4. before z2 successfully connected to z3, z3 get restarted with new - * epoch 0xb9. - * 5. z2 will retry around a few round (default 5s) before giving up, - * meanwhile it will report z3 as leader. - * 6. z1 restarted, and looking with peer epoch 0xb9. - * 7. z1 voted z3, and z3 was elected as leader again with peer epoch 0xb9. - * 8. z2 successfully connected to z3 before giving up, but with peer - * epoch 0xb8. - * 9. z1 get restarted, looking for leader with peer epoch 0xba, but cannot - * join, because z2 is reporting peer epoch 0xb8, while z3 is reporting - * 0xb9. - * - * By updating the election vote after actually following leader, we can - * avoid this kind of stuck happened. - * - * Btw, the zxid and electionEpoch could be inconsistent because of the same - * reason, it's better to update these as well after syncing with leader, but - * that required protocol change which is non trivial. This problem is worked - * around by skipping comparing the zxid and electionEpoch when counting for - * votes for out of election servers during looking for leader. - * - * See https://issues.apache.org/jira/browse/ZOOKEEPER-1732 - */ - protected void updateElectionVote(long newEpoch) { - Vote currentVote = getCurrentVote(); - if (currentVote != null) { - setCurrentVote(new Vote(currentVote.getId(), currentVote.getZxid(), currentVote.getElectionEpoch(), newEpoch, currentVote - .getState())); - } - } - - private void updateThreadName() { - String plain = cnxnFactory != null - ? cnxnFactory.getLocalAddress() != null - ? formatInetAddr(cnxnFactory.getLocalAddress()) - : "disabled" - : "disabled"; - String secure = secureCnxnFactory != null ? formatInetAddr(secureCnxnFactory.getLocalAddress()) : "disabled"; - setName(String.format("QuorumPeer[myid=%d](plain=%s)(secure=%s)", getMyId(), plain, secure)); - } - - /** - * Sets the time taken for leader election in milliseconds. - * - * @param electionTimeTaken time taken for leader election - */ - void setElectionTimeTaken(long electionTimeTaken) { - this.electionTimeTaken = electionTimeTaken; - } - - /** - * @return the time taken for leader election in milliseconds. - */ - long getElectionTimeTaken() { - return electionTimeTaken; - } - - void setQuorumServerSaslRequired(boolean serverSaslRequired) { - quorumServerSaslAuthRequired = serverSaslRequired; - LOG.info("{} set to {}", QuorumAuth.QUORUM_SERVER_SASL_AUTH_REQUIRED, serverSaslRequired); - } - - void setQuorumLearnerSaslRequired(boolean learnerSaslRequired) { - quorumLearnerSaslAuthRequired = learnerSaslRequired; - LOG.info("{} set to {}", QuorumAuth.QUORUM_LEARNER_SASL_AUTH_REQUIRED, learnerSaslRequired); - } - - void setQuorumSaslEnabled(boolean enableAuth) { - quorumSaslEnableAuth = enableAuth; - if (!quorumSaslEnableAuth) { - LOG.info("QuorumPeer communication is not secured! (SASL auth disabled)"); - } else { - LOG.info("{} set to {}", QuorumAuth.QUORUM_SASL_AUTH_ENABLED, enableAuth); - } - } - - void setQuorumServicePrincipal(String servicePrincipal) { - quorumServicePrincipal = servicePrincipal; - LOG.info("{} set to {}", QuorumAuth.QUORUM_KERBEROS_SERVICE_PRINCIPAL, quorumServicePrincipal); - } - - void setQuorumLearnerLoginContext(String learnerContext) { - quorumLearnerLoginContext = learnerContext; - LOG.info("{} set to {}", QuorumAuth.QUORUM_LEARNER_SASL_LOGIN_CONTEXT, quorumLearnerLoginContext); - } - - void setQuorumServerLoginContext(String serverContext) { - quorumServerLoginContext = serverContext; - LOG.info("{} set to {}", QuorumAuth.QUORUM_SERVER_SASL_LOGIN_CONTEXT, quorumServerLoginContext); - } - - void setQuorumCnxnThreadsSize(int qCnxnThreadsSize) { - if (qCnxnThreadsSize > QUORUM_CNXN_THREADS_SIZE_DEFAULT_VALUE) { - quorumCnxnThreadsSize = qCnxnThreadsSize; - } - LOG.info("quorum.cnxn.threads.size set to {}", quorumCnxnThreadsSize); - } - - boolean isQuorumSaslAuthEnabled() { - return quorumSaslEnableAuth; - } - - private boolean isQuorumServerSaslAuthRequired() { - return quorumServerSaslAuthRequired; - } - - private boolean isQuorumLearnerSaslAuthRequired() { - return quorumLearnerSaslAuthRequired; - } - - public QuorumCnxManager createCnxnManager() { - int timeout = quorumCnxnTimeoutMs > 0 ? quorumCnxnTimeoutMs : this.tickTime * this.syncLimit; - LOG.info("Using {}ms as the quorum cnxn socket timeout", timeout); - return new QuorumCnxManager( - this, - this.getMyId(), - this.getView(), - this.authServer, - this.authLearner, - timeout, - this.getQuorumListenOnAllIPs(), - this.quorumCnxnThreadsSize, - this.isQuorumSaslAuthEnabled()); - } - - boolean isLeader(long id) { - Vote vote = getCurrentVote(); - return vote != null && id == vote.getId(); - } - - public boolean isReconfigEnabled() { - return reconfigEnabled; - } - - @InterfaceAudience.Private - /** - * This is a metric that depends on the status of the peer. - */ public Integer getSynced_observers_metric() { - if (leader != null) { - return leader.getObservingLearners().size(); - } else if (follower != null) { - return follower.getSyncedObserverSize(); - } else { - return null; - } - } - - /** - * Create a new QuorumPeer and apply all the values per the already-parsed config. - * - * @param config The appertained quorum peer config. - * @return A QuorumPeer instantiated with specified peer config. Note this peer - * is not fully initialized; caller should finish initialization through - * additional configurations (connection factory settings, etc). - * - * @throws IOException - */ - public static QuorumPeer createFromConfig(QuorumPeerConfig config) throws IOException { - QuorumPeer quorumPeer = new QuorumPeer(); - quorumPeer.setTxnFactory(new FileTxnSnapLog(config.getDataLogDir(), config.getDataDir())); - quorumPeer.enableLocalSessions(config.areLocalSessionsEnabled()); - quorumPeer.enableLocalSessionsUpgrading(config.isLocalSessionsUpgradingEnabled()); - quorumPeer.setElectionType(config.getElectionAlg()); - quorumPeer.setMyid(config.getServerId()); - quorumPeer.setTickTime(config.getTickTime()); - quorumPeer.setMinSessionTimeout(config.getMinSessionTimeout()); - quorumPeer.setMaxSessionTimeout(config.getMaxSessionTimeout()); - quorumPeer.setInitLimit(config.getInitLimit()); - quorumPeer.setSyncLimit(config.getSyncLimit()); - quorumPeer.setConnectToLearnerMasterLimit(config.getConnectToLearnerMasterLimit()); - quorumPeer.setObserverMasterPort(config.getObserverMasterPort()); - quorumPeer.setConfigFileName(config.getConfigFilename()); - quorumPeer.setClientPortListenBacklog(config.getClientPortListenBacklog()); - quorumPeer.setZKDatabase(new ZKDatabase(quorumPeer.getTxnFactory())); - quorumPeer.setQuorumVerifier(config.getQuorumVerifier(), false); - if (config.getLastSeenQuorumVerifier() != null) { - quorumPeer.setLastSeenQuorumVerifier(config.getLastSeenQuorumVerifier(), false); - } - quorumPeer.initConfigInZKDatabase(); - quorumPeer.setSslQuorum(config.isSslQuorum()); - quorumPeer.setUsePortUnification(config.shouldUsePortUnification()); - quorumPeer.setLearnerType(config.getPeerType()); - quorumPeer.setSyncEnabled(config.getSyncEnabled()); - quorumPeer.setQuorumListenOnAllIPs(config.getQuorumListenOnAllIPs()); - if (config.sslQuorumReloadCertFiles) { - quorumPeer.getX509Util().enableCertFileReloading(); - } - quorumPeer.setMultiAddressEnabled(config.isMultiAddressEnabled()); - quorumPeer.setMultiAddressReachabilityCheckEnabled(config.isMultiAddressReachabilityCheckEnabled()); - quorumPeer.setMultiAddressReachabilityCheckTimeoutMs(config.getMultiAddressReachabilityCheckTimeoutMs()); - - // sets quorum sasl authentication configurations - quorumPeer.setQuorumSaslEnabled(config.quorumEnableSasl); - if (quorumPeer.isQuorumSaslAuthEnabled()) { - quorumPeer.setQuorumServerSaslRequired(config.quorumServerRequireSasl); - quorumPeer.setQuorumLearnerSaslRequired(config.quorumLearnerRequireSasl); - quorumPeer.setQuorumServicePrincipal(config.quorumServicePrincipal); - quorumPeer.setQuorumServerLoginContext(config.quorumServerLoginContext); - quorumPeer.setQuorumLearnerLoginContext(config.quorumLearnerLoginContext); - } - quorumPeer.setQuorumCnxnThreadsSize(config.quorumCnxnThreadsSize); - - if (config.jvmPauseMonitorToRun) { - quorumPeer.setJvmPauseMonitor(new JvmPauseMonitor(config)); - } - - return quorumPeer; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java deleted file mode 100644 index a96a395b03b..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.quorum; - -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Objects; -import java.util.stream.Collectors; -import org.apache.zookeeper.ZooDefs.OpCode; -import org.apache.zookeeper.jmx.MBeanRegistry; -import org.apache.zookeeper.server.DataTreeBean; -import org.apache.zookeeper.server.FinalRequestProcessor; -import org.apache.zookeeper.server.PrepRequestProcessor; -import org.apache.zookeeper.server.Request; -import org.apache.zookeeper.server.RequestProcessor; -import org.apache.zookeeper.server.ServerCnxn; -import org.apache.zookeeper.server.ZKDatabase; -import org.apache.zookeeper.server.ZooKeeperServer; -import org.apache.zookeeper.server.ZooKeeperServerBean; -import org.apache.zookeeper.server.persistence.FileTxnSnapLog; - -/** - * A ZooKeeperServer which comes into play when peer is partitioned from the - * majority. Handles read-only clients, but drops connections from not-read-only - * ones. - * <p> - * The very first processor in the chain of request processors is a - * ReadOnlyRequestProcessor which drops state-changing requests. - */ -public class ReadOnlyZooKeeperServer extends ZooKeeperServer { - - protected final QuorumPeer self; - private volatile boolean shutdown = false; - - ReadOnlyZooKeeperServer(FileTxnSnapLog logFactory, QuorumPeer self, ZKDatabase zkDb) { - super( - logFactory, - self.tickTime, - self.minSessionTimeout, - self.maxSessionTimeout, - self.clientPortListenBacklog, - zkDb, - self.getInitialConfig(), - self.isReconfigEnabled()); - this.self = self; - } - - @Override - protected void setupRequestProcessors() { - RequestProcessor finalProcessor = new FinalRequestProcessor(this); - RequestProcessor prepProcessor = new PrepRequestProcessor(this, finalProcessor); - ((PrepRequestProcessor) prepProcessor).start(); - firstProcessor = new ReadOnlyRequestProcessor(this, prepProcessor); - ((ReadOnlyRequestProcessor) firstProcessor).start(); - } - - @Override - public synchronized void startup() { - // check to avoid startup follows shutdown - if (shutdown) { - LOG.warn("Not starting Read-only server as startup follows shutdown!"); - return; - } - registerJMX(new ReadOnlyBean(this), self.jmxLocalPeerBean); - super.startup(); - self.setZooKeeperServer(this); - self.adminServer.setZooKeeperServer(this); - LOG.info("Read-only server started"); - } - - @Override - public void createSessionTracker() { - sessionTracker = new LearnerSessionTracker( - this, getZKDatabase().getSessionWithTimeOuts(), - this.tickTime, self.getMyId(), self.areLocalSessionsEnabled(), - getZooKeeperServerListener()); - } - - @Override - protected void startSessionTracker() { - ((LearnerSessionTracker) sessionTracker).start(); - } - - @Override - protected void setLocalSessionFlag(Request si) { - switch (si.type) { - case OpCode.createSession: - if (self.areLocalSessionsEnabled()) { - si.setLocalSession(true); - } - break; - case OpCode.closeSession: - if (((UpgradeableSessionTracker) sessionTracker).isLocalSession(si.sessionId)) { - si.setLocalSession(true); - } else { - LOG.warn("Submitting global closeSession request for session 0x{} in ReadOnly mode", - Long.toHexString(si.sessionId)); - } - break; - default: - break; - } - } - - @Override - protected void validateSession(ServerCnxn cnxn, long sessionId) throws IOException { - if (((LearnerSessionTracker) sessionTracker).isGlobalSession(sessionId)) { - String msg = "Refusing global session reconnection in RO mode " + cnxn.getRemoteSocketAddress(); - LOG.info(msg); - throw new ServerCnxn.CloseRequestException(msg, ServerCnxn.DisconnectReason.RENEW_GLOBAL_SESSION_IN_RO_MODE); - } - } - - @Override - protected void registerJMX() { - // register with JMX - try { - jmxDataTreeBean = new DataTreeBean(getZKDatabase().getDataTree()); - MBeanRegistry.getInstance().register(jmxDataTreeBean, jmxServerBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - jmxDataTreeBean = null; - } - } - - public void registerJMX(ZooKeeperServerBean serverBean, LocalPeerBean localPeerBean) { - // register with JMX - try { - jmxServerBean = serverBean; - MBeanRegistry.getInstance().register(serverBean, localPeerBean); - } catch (Exception e) { - LOG.warn("Failed to register with JMX", e); - jmxServerBean = null; - } - } - - @Override - protected void unregisterJMX() { - // unregister from JMX - try { - if (jmxDataTreeBean != null) { - MBeanRegistry.getInstance().unregister(jmxDataTreeBean); - } - } catch (Exception e) { - LOG.warn("Failed to unregister with JMX", e); - } - jmxDataTreeBean = null; - } - - protected void unregisterJMX(ZooKeeperServer zks) { - // unregister from JMX - try { - if (jmxServerBean != null) { - MBeanRegistry.getInstance().unregister(jmxServerBean); - } - } catch (Exception e) { - LOG.warn("Failed to unregister with JMX", e); - } - jmxServerBean = null; - } - - @Override - public String getState() { - return "read-only"; - } - - /** - * Returns the id of the associated QuorumPeer, which will do for a unique - * id of this server. - */ - @Override - public long getServerId() { - return self.getMyId(); - } - - @Override - public synchronized void shutdown(boolean fullyShutDown) { - if (!canShutdown()) { - LOG.debug("ZooKeeper server is not running, so not proceeding to shutdown!"); - } else { - shutdown = true; - unregisterJMX(this); - - // set peer's server to null - self.setZooKeeperServer(null); - // clear all the connections - self.closeAllConnections(); - - self.adminServer.setZooKeeperServer(null); - } - // shutdown the server itself - super.shutdown(fullyShutDown); - } - - @Override - public void dumpConf(PrintWriter pwriter) { - super.dumpConf(pwriter); - - pwriter.print("initLimit="); - pwriter.println(self.getInitLimit()); - pwriter.print("syncLimit="); - pwriter.println(self.getSyncLimit()); - pwriter.print("electionAlg="); - pwriter.println(self.getElectionType()); - pwriter.print("electionPort="); - pwriter.println(self.getElectionAddress().getAllPorts() - .stream().map(Objects::toString).collect(Collectors.joining("|"))); - pwriter.print("quorumPort="); - pwriter.println(self.getQuorumAddress().getAllPorts() - .stream().map(Objects::toString).collect(Collectors.joining("|"))); - pwriter.print("peerType="); - pwriter.println(self.getLearnerType().ordinal()); - } - - @Override - protected void setState(State state) { - this.state = state; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/SendAckRequestProcessor.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/SendAckRequestProcessor.java deleted file mode 100644 index d65ead216f0..00000000000 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/SendAckRequestProcessor.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.quorum; - -import java.io.Flushable; -import java.io.IOException; -import java.net.Socket; -import org.apache.zookeeper.ZooDefs.OpCode; -import org.apache.zookeeper.server.Request; -import org.apache.zookeeper.server.RequestProcessor; -import org.apache.zookeeper.server.ServerMetrics; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SendAckRequestProcessor implements RequestProcessor, Flushable { - - private static final Logger LOG = LoggerFactory.getLogger(SendAckRequestProcessor.class); - - final Learner learner; - - SendAckRequestProcessor(Learner peer) { - this.learner = peer; - } - - public void processRequest(Request si) { - if (si.type != OpCode.sync) { - QuorumPacket qp = new QuorumPacket(Leader.ACK, si.getHdr().getZxid(), null, null); - try { - si.logLatency(ServerMetrics.getMetrics().PROPOSAL_ACK_CREATION_LATENCY); - - learner.writePacket(qp, false); - } catch (IOException e) { - LOG.warn("Closing connection to leader, exception during packet send", e); - try { - if (!learner.sock.isClosed()) { - learner.sock.close(); - } - } catch (IOException e1) { - // Nothing to do, we are shutting things down, so an exception here is irrelevant - LOG.debug("Ignoring error closing the connection", e1); - } - } - } - } - - public void flush() throws IOException { - try { - learner.writePacket(null, true); - } catch (IOException e) { - LOG.warn("Closing connection to leader, exception during packet send", e); - try { - Socket socket = learner.sock; - if (socket != null && !socket.isClosed()) { - learner.sock.close(); - } - } catch (IOException e1) { - // Nothing to do, we are shutting things down, so an exception here is irrelevant - LOG.debug("Ignoring error closing the connection", e1); - } - } - } - - public void shutdown() { - // Nothing needed - } - -} diff --git a/zookeeper-server/zookeeper-server/CMakeLists.txt b/zookeeper-server/zookeeper-server/CMakeLists.txt index c7e9679ed24..30ed1ed4404 100644 --- a/zookeeper-server/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/zookeeper-server/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar(zookeeper-server-3.9.1-jar-with-dependencies.jar) +install_jar(zookeeper-server-3.9.2-jar-with-dependencies.jar) # Make symlink so that we have a default version, should be done only in zookeeper-server module -install_symlink(lib/jars/zookeeper-server-3.9.1-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) +install_symlink(lib/jars/zookeeper-server-3.9.2-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server/pom.xml b/zookeeper-server/zookeeper-server/pom.xml index f1b33dd0ae7..791c026234a 100644 --- a/zookeeper-server/zookeeper-server/pom.xml +++ b/zookeeper-server/zookeeper-server/pom.xml @@ -8,11 +8,11 @@ <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>zookeeper-server-3.9.1</artifactId> + <artifactId>zookeeper-server-3.9.2</artifactId> <packaging>container-plugin</packaging> <version>8-SNAPSHOT</version> <properties> - <zookeeper.version>3.9.1</zookeeper.version> + <zookeeper.version>3.9.2</zookeeper.version> </properties> <dependencies> <dependency> diff --git a/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java index 891a35582b3..c74a020bcf4 100644 --- a/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java @@ -26,6 +26,7 @@ public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); + @SuppressWarnings("try") @Override public void reconfigure(String connectionSpec, String servers) throws ReconfigException { diff --git a/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java index 895bbeffa5f..00af31b46d4 100644 --- a/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java @@ -384,13 +384,13 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { + " minSessionTimeout {} ms" + " maxSessionTimeout {} ms" + " clientPortListenBacklog {}" - + " datadir {}" + + " dataLogdir {}" + " snapdir {}", tickTime, getMinSessionTimeout(), getMaxSessionTimeout(), getClientPortListenBacklog(), - txnLogFactory.getDataDir(), + txnLogFactory.getDataLogDir(), txnLogFactory.getSnapDir()); } @@ -442,7 +442,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { pwriter.print("dataDirSize="); pwriter.println(getDataDirSize()); pwriter.print("dataLogDir="); - pwriter.println(zkDb.snapLog.getDataDir().getAbsolutePath()); + pwriter.println(zkDb.snapLog.getDataLogDir().getAbsolutePath()); pwriter.print("dataLogSize="); pwriter.println(getLogDirSize()); pwriter.print("tickTime="); @@ -464,7 +464,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { return new ZooKeeperServerConf( getClientPort(), zkDb.snapLog.getSnapDir().getAbsolutePath(), - zkDb.snapLog.getDataDir().getAbsolutePath(), + zkDb.snapLog.getDataLogDir().getAbsolutePath(), getTickTime(), getMaxClientCnxnsPerHost(), getMinSessionTimeout(), @@ -649,7 +649,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { if (zkDb == null) { return 0L; } - File path = zkDb.snapLog.getDataDir(); + File path = zkDb.snapLog.getSnapDir(); return getDirSize(path); } @@ -658,7 +658,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { if (zkDb == null) { return 0L; } - File path = zkDb.snapLog.getSnapDir(); + File path = zkDb.snapLog.getDataLogDir(); return getDirSize(path); } @@ -1506,7 +1506,9 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { throw new CloseRequestException(msg, ServerCnxn.DisconnectReason.NOT_READ_ONLY_CLIENT); } if (request.getLastZxidSeen() > zkDb.dataTree.lastProcessedZxid) { - String msg = "Refusing session request for client " + String msg = "Refusing session(0x" + + Long.toHexString(sessionId) + + ") request for client " + cnxn.getRemoteSocketAddress() + " as it has seen zxid 0x" + Long.toHexString(request.getLastZxidSeen()) diff --git a/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java index 8d8b6dabce8..3c7b2148400 100644 --- a/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java +++ b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java @@ -597,7 +597,6 @@ public class Learner { willSnapshot = false; // but anything after this needs to go to the transaction log; or } - self.setCurrentEpoch(newEpoch); sock.setSoTimeout(self.tickTime * self.syncLimit); self.setSyncMode(QuorumPeer.SyncMode.NONE); zk.startupWithoutServing(); @@ -613,6 +612,8 @@ public class Learner { delayedProposals.clear(); fzk.syncProcessor.syncFlush(); } + + self.setCurrentEpoch(newEpoch); } void flushAcks() throws InterruptedException { |