diff options
| author | Håkon Hallingstad <hakon@yahooinc.com> | 2023-09-25 22:40:34 +0200 |
|---|---|---|
| committer | Håkon Hallingstad <hakon@yahooinc.com> | 2023-09-25 22:40:34 +0200 |
| commit | d62924c2ae44658c0ba7ab6b008bc17523fe4a35 (patch) | |
| tree | 01561b32e4838606c18617bab8a657e045d97f4d | |
| parent | ad8821002334cec2110df06aebae2906b7b0128b (diff) | |
| parent | cdb6e9f363e5b88577d4df3fb98a74ae5399eb14 (diff) | |
Merge branch 'master' into hakonhall/add-ipv6-to-public-zonal-endpoints-in-gcp-2nd-attempt
106 files changed, 1696 insertions, 610 deletions
diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 1c8f628fe18..572c3095570 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -24,9 +24,9 @@ chalk "^2.4.2" "@babel/compat-data@^7.22.9": - version "7.22.9" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.22.9.tgz#71cdb00a1ce3a329ce4cbec3a44f9fef35669730" - integrity sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ== + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.22.20.tgz#8df6e96661209623f1975d66c35ffca66f3306d0" + integrity sha512-BQYjKbpXjoXwFW5jGqiizJQQT/aC7pFm9Ok1OWssonuguICi264lbgMzRp2ZMmRSlfkX6DsWDDcsrctK8Rwfiw== "@babel/core@^7.1.0", "@babel/core@^7.12.17": version "7.22.9" @@ -70,21 +70,21 @@ json5 "^2.2.3" semver "^6.3.1" -"@babel/core@^7.22.9": - version "7.22.11" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.11.tgz#8033acaa2aa24c3f814edaaa057f3ce0ba559c24" - integrity sha512-lh7RJrtPdhibbxndr6/xx0w8+CVlY5FJZiaSz908Fpy+G0xkBFTvwLcKJFF4PJxVfGhVWNebikpWGnOoC71juQ== +"@babel/core@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.20.tgz#e3d0eed84c049e2a2ae0a64d27b6a37edec385b7" + integrity sha512-Y6jd1ahLubuYweD/zJH+vvOY141v4f9igNQAQ+MBgq9JlHS2iTsZKn1aMsb3vGccZsXI16VzTBw52Xx0DWmtnA== dependencies: "@ampproject/remapping" "^2.2.0" - "@babel/code-frame" "^7.22.10" - "@babel/generator" "^7.22.10" - "@babel/helper-compilation-targets" "^7.22.10" - "@babel/helper-module-transforms" "^7.22.9" - "@babel/helpers" "^7.22.11" - "@babel/parser" "^7.22.11" - "@babel/template" "^7.22.5" - "@babel/traverse" "^7.22.11" - "@babel/types" "^7.22.11" + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.22.15" + "@babel/helper-compilation-targets" "^7.22.15" + "@babel/helper-module-transforms" "^7.22.20" + "@babel/helpers" "^7.22.15" + "@babel/parser" "^7.22.16" + "@babel/template" "^7.22.15" + "@babel/traverse" "^7.22.20" + "@babel/types" "^7.22.19" convert-source-map "^1.7.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -111,7 +111,7 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" -"@babel/helper-compilation-targets@^7.22.10", "@babel/helper-compilation-targets@^7.22.15": +"@babel/helper-compilation-targets@^7.22.15": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz#0698fc44551a26cf29f18d4662d5bf545a6cfc52" integrity sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw== @@ -133,10 +133,10 @@ lru-cache "^5.1.1" semver "^6.3.1" -"@babel/helper-environment-visitor@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz#f06dd41b7c1f44e1f8da6c4055b41ab3a09a7e98" - integrity sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q== +"@babel/helper-environment-visitor@^7.22.20", "@babel/helper-environment-visitor@^7.22.5": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== "@babel/helper-function-name@^7.22.5": version "7.22.5" @@ -178,6 +178,17 @@ "@babel/helper-split-export-declaration" "^7.22.6" "@babel/helper-validator-identifier" "^7.22.15" +"@babel/helper-module-transforms@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.22.20.tgz#da9edc14794babbe7386df438f3768067132f59e" + integrity sha512-dLT7JVWIUUxKOs1UnJUBR3S70YK+pKX6AbJgB2vMIvEkZkrfJDbYDJesnPshtKV4LhDOR3Oc5YULeDizRek+5A== + dependencies: + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-module-imports" "^7.22.15" + "@babel/helper-simple-access" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/helper-validator-identifier" "^7.22.20" + "@babel/helper-module-transforms@^7.22.5": version "7.22.9" resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.22.9.tgz#92dfcb1fbbb2bc62529024f72d942a8c97142129" @@ -213,17 +224,17 @@ resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== -"@babel/helper-validator-identifier@^7.22.15", "@babel/helper-validator-identifier@^7.22.5": - version "7.22.15" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.15.tgz#601fa28e4cc06786c18912dca138cec73b882044" - integrity sha512-4E/F9IIEi8WR94324mbDUMo074YTheJmd7eZF5vITTeYchqAi6sYXRLHUVsmkdmY4QjfKTcB2jB7dVP3NaBElQ== +"@babel/helper-validator-identifier@^7.22.15", "@babel/helper-validator-identifier@^7.22.19", "@babel/helper-validator-identifier@^7.22.20", "@babel/helper-validator-identifier@^7.22.5": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== "@babel/helper-validator-option@^7.22.15", "@babel/helper-validator-option@^7.22.5": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz#694c30dfa1d09a6534cdfcafbe56789d36aba040" integrity sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA== -"@babel/helpers@^7.22.11", "@babel/helpers@^7.22.15": +"@babel/helpers@^7.22.15": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.22.15.tgz#f09c3df31e86e3ea0b7ff7556d85cdebd47ea6f1" integrity sha512-7pAjK0aSdxOwR+CcYAqgWOGy5dcfvzsTIfFTb2odQqW47MDfv14UaJDY6eng8ylM2EaeKXdxaSWESbkmaQHTmw== @@ -242,11 +253,11 @@ "@babel/types" "^7.22.11" "@babel/highlight@^7.22.13": - version "7.22.13" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.13.tgz#9cda839e5d3be9ca9e8c26b6dd69e7548f0cbf16" - integrity sha512-C/BaXcnnvBCmHTpz/VGZ8jgtE2aYlW4hxDhseJAWZb7gqGM/qtCK6iZUb0TyKFf7BOUsBH7Q7fkRsDRhg1XklQ== + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.20.tgz#4ca92b71d80554b01427815e06f2df965b9c1f54" + integrity sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg== dependencies: - "@babel/helper-validator-identifier" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" chalk "^2.4.2" js-tokens "^4.0.0" @@ -404,7 +415,7 @@ "@babel/parser" "^7.22.15" "@babel/types" "^7.22.15" -"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.15", "@babel/traverse@^7.22.17": +"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.17": version "7.22.17" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.17.tgz#b23c203ab3707e3be816043081b4a994fcacec44" integrity sha512-xK4Uwm0JnAMvxYZxOVecss85WxTEIbTa7bnGyf/+EgCL5Zt3U7htUpEOWv9detPlamGKuRzCqw74xVglDWpPdg== @@ -420,6 +431,22 @@ debug "^4.1.0" globals "^11.1.0" +"@babel/traverse@^7.22.15", "@babel/traverse@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.20.tgz#db572d9cb5c79e02d83e5618b82f6991c07584c9" + integrity sha512-eU260mPZbU7mZ0N+X10pxXhQFMGTeLb9eFS0mxehS8HZp9o1uSnFeWQuG1UPrlxgA7QoUzFhOnilHDp0AXCyHw== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.22.15" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.22.5" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.22.16" + "@babel/types" "^7.22.19" + debug "^4.1.0" + globals "^11.1.0" + "@babel/traverse@^7.22.8": version "7.22.11" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.11.tgz#71ebb3af7a05ff97280b83f05f8865ac94b2027c" @@ -436,7 +463,16 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.10", "@babel/types@^7.22.11", "@babel/types@^7.22.15", "@babel/types@^7.22.17", "@babel/types@^7.22.5", "@babel/types@^7.3.3": +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.17", "@babel/types@^7.22.19", "@babel/types@^7.22.5": + version "7.22.19" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.19.tgz#7425343253556916e440e662bb221a93ddb75684" + integrity sha512-P7LAw/LbojPzkgp5oznjE6tQEIWbp4PkkfrZDINTro9zgBRtI324/EYsiSI7lhPbpIQ+DCeR2NNmMWANGGfZsg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.19" + to-fast-properties "^2.0.0" + +"@babel/types@^7.22.10", "@babel/types@^7.22.11", "@babel/types@^7.3.3": version "7.22.17" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.17.tgz#f753352c4610ffddf9c8bc6823f9ff03e2303eee" integrity sha512-YSQPHLFtQNE5xN9tHuZnzu8vPr61wVTBZdfv1meex1NBosa4iT05k/Jw06ddJugi4bk7The/oSwQGFcksmEJQg== @@ -1244,22 +1280,40 @@ "@types/babel__template" "*" "@types/babel__traverse" "*" +"@types/babel__core@^7.20.2": + version "7.20.2" + resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.2.tgz#215db4f4a35d710256579784a548907237728756" + integrity sha512-pNpr1T1xLUc2l3xJKuPtsEky3ybxN3m4fJkknfIpTCTfIZCDW57oAg+EfCgIIp2rvCe0Wn++/FfodDS4YXxBwA== + dependencies: + "@babel/parser" "^7.20.7" + "@babel/types" "^7.20.7" + "@types/babel__generator" "*" + "@types/babel__template" "*" + "@types/babel__traverse" "*" + "@types/babel__generator@*": - version "7.6.4" - resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.4.tgz#1f20ce4c5b1990b37900b63f050182d28c2439b7" - integrity sha512-tFkciB9j2K755yrTALxD44McOrk+gfpIpvC3sxHjRawj6PfnQxrse4Clq5y/Rq+G3mrBurMax/lG8Qn2t9mSsg== + version "7.6.5" + resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.5.tgz#281f4764bcbbbc51fdded0f25aa587b4ce14da95" + integrity sha512-h9yIuWbJKdOPLJTbmSpPzkF67e659PbQDba7ifWm5BJ8xTv+sDmS7rFmywkWOvXedGTivCdeGSIIX8WLcRTz8w== dependencies: "@babel/types" "^7.0.0" "@types/babel__template@*": - version "7.4.1" - resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.1.tgz#3d1a48fd9d6c0edfd56f2ff578daed48f36c8969" - integrity sha512-azBFKemX6kMg5Io+/rdGT0dkGreboUVR0Cdm3fz9QJWpaQGJRQXl7C+6hOTCZcMll7KFyEQpgbYI2lHdsS4U7g== + version "7.4.2" + resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.2.tgz#843e9f1f47c957553b0c374481dc4772921d6a6b" + integrity sha512-/AVzPICMhMOMYoSx9MoKpGDKdBRsIXMNByh1PXSZoa+v6ZoLa8xxtsT/uLQ/NJm0XVAWl/BvId4MlDeXJaeIZQ== dependencies: "@babel/parser" "^7.1.0" "@babel/types" "^7.0.0" -"@types/babel__traverse@*", "@types/babel__traverse@^7.0.6": +"@types/babel__traverse@*": + version "7.20.2" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.2.tgz#4ddf99d95cfdd946ff35d2b65c978d9c9bf2645d" + integrity sha512-ojlGK1Hsfce93J0+kn3H5R73elidKUaZonirN33GSmgTUMpzI/MIFfSpF3haANe3G1bEBS9/9/QEqwTzwqFsKw== + dependencies: + "@babel/types" "^7.20.7" + +"@types/babel__traverse@^7.0.6": version "7.20.1" resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.1.tgz#dd6f1d2411ae677dcb2db008c962598be31d6acf" integrity sha512-MitHFXnhtgwsGZWtT68URpOvLN4EREih1u3QtQiN4VdAxWKRVvGCSvw/Qth0M0Qq3pJpnGOu5JaM/ydK7OGbqg== @@ -1337,13 +1391,14 @@ "@types/yargs-parser" "*" "@vitejs/plugin-react@^4": - version "4.0.4" - resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.0.4.tgz#31c3f779dc534e045c4b134e7cf7b150af0a7646" - integrity sha512-7wU921ABnNYkETiMaZy7XqpueMnpu5VxvVps13MjmCo+utBdD79sZzrApHawHtVX66cCJQQTXFcjH0y9dSUK8g== + version "4.1.0" + resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.1.0.tgz#e4f56f46fd737c5d386bb1f1ade86ba275fe09bd" + integrity sha512-rM0SqazU9iqPUraQ2JlIvReeaxOoRj6n+PzB1C0cBzIbd8qP336nC39/R9yPi3wVcah7E7j/kdU1uCUqMEU4OQ== dependencies: - "@babel/core" "^7.22.9" + "@babel/core" "^7.22.20" "@babel/plugin-transform-react-jsx-self" "^7.22.5" "@babel/plugin-transform-react-jsx-source" "^7.22.5" + "@types/babel__core" "^7.20.2" react-refresh "^0.14.0" acorn-jsx@^5.3.2: @@ -1725,14 +1780,14 @@ braces@^3.0.2: fill-range "^7.0.1" browserslist@^4.21.9: - version "4.21.10" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.10.tgz#dbbac576628c13d3b2231332cb2ec5a46e015bb0" - integrity sha512-bipEBdZfVH5/pwrvqc+Ub0kUPVfGUhlKxbvfD+z1BDnPEO/X98ruXGA1WP5ASpAFKan7Qr6j736IacbZQuAlKQ== + version "4.21.11" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.11.tgz#35f74a3e51adc4d193dcd76ea13858de7b8fecb8" + integrity sha512-xn1UXOKUz7DjdGlg9RrUr0GGiWzI97UQJnugHtH0OLDfJB7jMgoIkYvRIEO1l9EeEERVqeqLYOcFBW9ldjypbQ== dependencies: - caniuse-lite "^1.0.30001517" - electron-to-chromium "^1.4.477" + caniuse-lite "^1.0.30001538" + electron-to-chromium "^1.4.526" node-releases "^2.0.13" - update-browserslist-db "^1.0.11" + update-browserslist-db "^1.0.13" bser@2.1.1: version "2.1.1" @@ -1791,10 +1846,10 @@ camelcase@^6.2.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -caniuse-lite@^1.0.30001517: - version "1.0.30001533" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001533.tgz#1180daeb2518b93c82f19b904d1fefcf82197707" - integrity sha512-9aY/b05NKU4Yl2sbcJhn4A7MsGwR1EPfW/nrqsnqVA0Oq50wpmPaGI+R1Z0UKlUl96oxUkGEOILWtOHck0eCWw== +caniuse-lite@^1.0.30001538: + version "1.0.30001539" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001539.tgz#325a387ab1ed236df2c12dc6cd43a4fff9903a44" + integrity sha512-hfS5tE8bnNiNvEOEkm8HElUHroYwlqMMENEzELymy77+tJ6m+gA2krtHl5hxJaj71OlpC2cHZbdSMX1/YEqEkA== capture-exit@^2.0.0: version "2.0.0" @@ -2124,10 +2179,10 @@ dom-helpers@^5.0.1: "@babel/runtime" "^7.8.7" csstype "^3.0.2" -electron-to-chromium@^1.4.477: - version "1.4.515" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.515.tgz#f5fec9662106ac5752894af221606cf4db443e70" - integrity sha512-VTq6vjk3kCfG2qdzQRd/i9dIyVVm0dbtZIgFzrLgfB73mXDQT2HPKVRc1EoZcAVUv9XhXAu08DWqJuababdGGg== +electron-to-chromium@^1.4.526: + version "1.4.528" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.528.tgz#7c900fd73d9d2e8bb0dab0e301f25f0f4776ef2c" + integrity sha512-UdREXMXzLkREF4jA8t89FQjA8WHI6ssP38PMY4/4KhXFQbtImnghh4GkCgrtiZwLKUKVD2iTVXvDVQjfomEQuA== emittery@^0.13.1: version "0.13.1" @@ -5366,10 +5421,10 @@ untildify@^4.0.0: resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== -update-browserslist-db@^1.0.11: - version "1.0.11" - resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz#9a2a641ad2907ae7b3616506f4b977851db5b940" - integrity sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA== +update-browserslist-db@^1.0.13: + version "1.0.13" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4" + integrity sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg== dependencies: escalade "^3.1.1" picocolors "^1.0.0" diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java index 6b0adb5d079..2dbbc8a5820 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java @@ -5,13 +5,20 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import java.util.logging.Level; -import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.yolean.Exceptions; -import java.io.*; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -208,6 +215,8 @@ public class FilesApplicationFile extends ApplicationFile { } } + @Override public long getSize() { return file.length(); } + @Override public int compareTo(ApplicationFile other) { if (other == this) return 0; diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java index a55ae795d28..97336b2bca0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java @@ -160,6 +160,8 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract MetaData getMetaData(); + public abstract long getSize(); + public static class MetaData { public String status = "unknown"; diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java index d9ebd902e3e..65e6bc2803a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; +import java.util.function.Supplier; import java.util.logging.Level; /** @@ -13,6 +14,10 @@ public interface DeployLogger { /** Log a message unrelated to the application package, e.g. internal error/status. */ void log(Level level, String message); + default void log(Level level, Supplier<String> message) { log(level, message.get()); } + + default void log(Level level, Supplier<String> message, Throwable throwable) { log(level, message); } + /** * Log a message related to the application package. These messages should be actionable by the user, f.ex. to * signal usage of invalid/deprecated syntax diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java index 69749ee6f96..0276985d6a6 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java @@ -168,13 +168,7 @@ public class ApplicationClusterEndpoint { return name; } - public static DnsName sharedNameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { - String name = dnsParts(systemName, cluster, applicationId) - .filter(Objects::nonNull) // remove null values that were "default" - .collect(Collectors.joining("--")); - return new DnsName(sanitize(name) + suffix); // Need to sanitize name since it is considered one label - } - + // TODO(mpolden): Remove when config-models < 8.232 are gone public static DnsName sharedL4NameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { String name = dnsParts(systemName, cluster, applicationId) .filter(Objects::nonNull) // remove null values that were "default" diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 37b24f0ac1d..446c32801e0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -117,6 +117,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean enableNestedMultivalueGrouping() { return false; } @ModelFeatureFlag(owners = {"jonmv"}) default boolean useReconfigurableDispatcher() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default int contentLayerMetadataFeatureLevel() { return 0; } + @ModelFeatureFlag(owners = {"bjorncs"}) default boolean dynamicHeapSize() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java new file mode 100644 index 00000000000..9507f7db7b6 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.config.model.api; + +import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; + +/** + * @author bjorncs + */ +public interface OnnxModelCost { + + default Calculator newCalculator(DeployLogger logger) { return newCalculator(null, logger); } + Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger); + + interface Calculator { + long aggregatedModelCostInBytes(); + void registerModel(ApplicationFile path); + void registerModel(ModelReference ref); + } + + static OnnxModelCost disabled() { + return (__, ___) -> new Calculator() { + @Override public long aggregatedModelCostInBytes() { return 0; } + @Override public void registerModel(ApplicationFile path) {} + @Override public void registerModel(ModelReference ref) {} + }; + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 4df7a76031a..5ab258ecce8 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -18,6 +18,7 @@ import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.api.Reindexing; import com.yahoo.config.model.api.ValidationParameters; @@ -90,6 +91,7 @@ public class DeployState implements ConfigDefinitionStore { private final Provisioned provisioned; private final Reindexing reindexing; private final ExecutorService executor; + private final OnnxModelCost onnxModelCost; public static DeployState createTestState() { return new Builder().build(); @@ -124,7 +126,8 @@ public class DeployState implements ConfigDefinitionStore { boolean accessLoggingEnabledByDefault, Optional<DockerImage> wantedDockerImageRepo, Reindexing reindexing, - Optional<ValidationOverrides> validationOverrides) { + Optional<ValidationOverrides> validationOverrides, + OnnxModelCost onnxModelCost) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.executor = executor; @@ -152,6 +155,7 @@ public class DeployState implements ConfigDefinitionStore { this.now = now; this.wantedDockerImageRepo = wantedDockerImageRepo; this.reindexing = reindexing; + this.onnxModelCost = onnxModelCost; } public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { @@ -305,6 +309,8 @@ public class DeployState implements ConfigDefinitionStore { public Optional<Reindexing> reindexing() { return Optional.ofNullable(reindexing); } + public OnnxModelCost onnxModelCost() { return onnxModelCost; } + public boolean isHostedTenantApplication(ApplicationType type) { boolean isTesterApplication = getProperties().applicationId().instance().isTester(); return isHosted() && type == ApplicationType.DEFAULT && !isTesterApplication; @@ -333,6 +339,7 @@ public class DeployState implements ConfigDefinitionStore { private QueryProfiles queryProfiles = null; private Reindexing reindexing = null; private Optional<ValidationOverrides> validationOverrides = Optional.empty(); + private OnnxModelCost onnxModelCost = OnnxModelCost.disabled(); public Builder() {} @@ -450,6 +457,8 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder onnxModelCost(OnnxModelCost instance) { this.onnxModelCost = instance; return this; } + public DeployState build() { return build(new ValidationParameters()); } @@ -482,7 +491,8 @@ public class DeployState implements ConfigDefinitionStore { accessLoggingEnabledByDefault, wantedDockerImageRepo, reindexing, - validationOverrides); + validationOverrides, + onnxModelCost); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 815c32e3c8f..77356292f9a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -86,6 +86,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean allowUserFilters = true; private List<DataplaneToken> dataplaneTokens; private int contentLayerMetadataFeatureLevel = 0; + private boolean dynamicHeapSize = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -144,6 +145,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only @Override public List<DataplaneToken> dataplaneTokens() { return dataplaneTokens; } @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } + @Override public boolean dynamicHeapSize() { return dynamicHeapSize; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -379,6 +381,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setDynamicHeapSize(boolean b) { this.dynamicHeapSize = b; return this; } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index dbcd1cea2fa..342b5f243e7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -488,6 +488,8 @@ public class MockApplicationPackage implements ApplicationPackage { throw new UnsupportedOperationException(); } + @Override public long getSize() { return file.length(); } + @Override public int compareTo(ApplicationFile other) { return this.getPath().getName().compareTo((other).getPath().getName()); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index 7d88985b2d5..f6a022e9930 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -82,7 +82,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } // Commands for summary fields - // TODO: Move to fieldinfo and implement differently. This is not right + // TODO: Move to schemainfo and implement differently for (SummaryField summaryField : schema.getUniqueNamedSummaryFields().values()) { if (summaryField.getTransform().isTeaser()) { addIndexCommand(summaryField.getName(), CMD_DYNTEASER); @@ -90,6 +90,13 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { if (summaryField.getTransform().isBolded()) { addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT); } + + var sourceField = schema.getField(summaryField.getSourceField()); // Take the first as they should all be consistent + if (sourceField != null && sourceField.getMatching().getType().equals(MatchType.GRAM)) { + addIndexCommand(summaryField.getName(), + "ngram " + (sourceField.getMatching().getGramSize().orElse(NGramMatch.DEFAULT_GRAM_SIZE))); + + } } } @@ -452,7 +459,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { iiB.command( new IndexInfoConfig.Indexinfo.Command.Builder() .indexname(fieldSet.getName()) - .command("ngram "+(fieldSetMatching.getGramSize()>0 ? fieldSetMatching.getGramSize() : NGramMatch.DEFAULT_GRAM_SIZE))); + .command("ngram " + fieldSetMatching.getGramSize().orElse(NGramMatch.DEFAULT_GRAM_SIZE))); } else if (fieldSetMatching.getType().equals(MatchType.TEXT)) { } diff --git a/config-model/src/main/java/com/yahoo/schema/document/Matching.java b/config-model/src/main/java/com/yahoo/schema/document/Matching.java index 1fe947d672b..264fd0ff3b9 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Matching.java @@ -1,7 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.document; +import com.yahoo.schema.processing.NGramMatch; + import java.io.Serializable; +import java.util.OptionalInt; /** * Defines how a field should be matched. @@ -23,8 +26,8 @@ public class Matching implements Cloneable, Serializable { private boolean algorithmUserSet = false; - /** The gram size is the n in n-gram, or -1 if not set. Should only be set with gram matching. */ - private int gramSize = -1; + /** The gram size is the n in n-gram, or empty if not set. Should only be set with gram matching. */ + private OptionalInt gramSize = OptionalInt.empty(); /** Maximum number of characters to consider when searching in this field. Used for limiting resources, especially in streaming search. */ private Integer maxLength; @@ -67,10 +70,10 @@ public class Matching implements Cloneable, Serializable { public boolean isSuffix() { return algorithm == MatchAlgorithm.SUFFIX; } - /** Returns the gram size, or -1 if not set. Should only be set with gram matching. */ - public int getGramSize() { return gramSize; } + /** Returns the gram size, or empty if not set. Should only be set with gram matching. */ + public OptionalInt getGramSize() { return gramSize; } - public void setGramSize(int gramSize) { this.gramSize=gramSize; } + public void setGramSize(int gramSize) { this.gramSize = OptionalInt.of(gramSize); } /** * Merge data from another matching object @@ -107,10 +110,11 @@ public class Matching implements Cloneable, Serializable { @Override public String toString() { - return type + " matching [" + (type==MatchType.GRAM ? "gram size " + gramSize : "supports " + algorithm) + - "], [exact-terminator "+exactMatchTerminator+"]"; + return type + " matching [" + (type == MatchType.GRAM ? "gram size " + gramSize.orElse(NGramMatch.DEFAULT_GRAM_SIZE) : "supports " + algorithm) + + "], [exact-terminator " + exactMatchTerminator + "]"; } + @Override public Matching clone() { try { return (Matching)super.clone(); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java index f1ff910be43..6ec5428156f 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java @@ -31,7 +31,7 @@ public class NGramMatch extends Processor { for (SDField field : schema.allConcreteFields()) { if (field.getMatching().getType().equals(MatchType.GRAM)) implementGramMatch(schema, field, validate); - else if (validate && field.getMatching().getGramSize() >= 0) + else if (validate && field.getMatching().getGramSize().isPresent()) throw new IllegalArgumentException("gram-size can only be set when the matching mode is 'gram'"); } } @@ -40,9 +40,7 @@ public class NGramMatch extends Processor { if (validate && field.doesAttributing() && ! field.doesIndexing()) throw new IllegalArgumentException("gram matching is not supported with attributes, use 'index' in indexing"); - int n = field.getMatching().getGramSize(); - if (n < 0) - n = DEFAULT_GRAM_SIZE; // not set - use default gram size + int n = field.getMatching().getGramSize().orElse(DEFAULT_GRAM_SIZE); if (validate && n == 0) throw new IllegalArgumentException("Illegal gram size in " + field + ": Must be at least 1"); field.getNormalizing().inferCodepoint(); diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 7439e65dee6..49cd36e4bc2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -17,9 +17,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; */ public class SummaryField extends Field implements Cloneable, TypedKey { - /** - * A source (field name). - */ + /** A source (field name). */ public static class Source implements Serializable { private final String name; @@ -38,12 +36,8 @@ public class SummaryField extends Field implements Cloneable, TypedKey { @Override public boolean equals(Object obj) { - if (!(obj instanceof Source)) { - return false; - } - Source other = (Source)obj; - return name.equals(other.name) && - override == other.override; + if (!(obj instanceof Source other)) return false; + return name.equals(other.name) && override == other.override; } @Override @@ -67,14 +61,14 @@ public class SummaryField extends Field implements Cloneable, TypedKey { */ private Set<Source> sources = new java.util.LinkedHashSet<>(); - private Set<String> destinations=new java.util.LinkedHashSet<>(); + private Set<String> destinations =new java.util.LinkedHashSet<>(); /** True if this field was defined implicitly */ - private boolean implicit=false; + private boolean implicit = false; /** Creates a summary field with NONE as transform */ public SummaryField(String name, DataType type) { - this(name,type, SummaryTransform.NONE); + this(name, type, SummaryTransform.NONE); } /** Creates a summary field with NONE as transform */ @@ -97,7 +91,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { public boolean isImplicit() { return implicit; } public void setTransform(SummaryTransform transform) { - this.transform=transform; + this.transform = transform; if (SummaryTransform.DYNAMICTEASER.equals(transform) || SummaryTransform.BOLDED.equals(transform)) { // This is the kind of logic we want to have in processing, // but can't because of deriveDocuments mode, which doesn't run @@ -110,9 +104,9 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** Returns the first source field of this, or null if the source field is not present */ public String getSourceField() { - String sourceName=getName(); - if (sources.size()>0) - sourceName=sources.iterator().next().getName(); + String sourceName = getName(); + if ( ! sources.isEmpty()) + sourceName = sources.iterator().next().getName(); return sourceName; } @@ -137,7 +131,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** Returns the first source name of this, or the field name if no source has been set */ public String getSingleSource() { - if (sources.size()==0) return getName(); + if (sources.isEmpty()) return getName(); return sources.iterator().next().getName(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java b/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java new file mode 100644 index 00000000000..fddf8409376 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java @@ -0,0 +1,120 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model; + +import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.vespa.model.ml.OnnxModelProbe; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * Aggregates estimated footprint of configured ONNX models. + * + * @author bjorncs + */ +public class DefaultOnnxModelCost implements OnnxModelCost { + + @Override + public Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger) { + return new CalculatorImpl(appPkg, logger); + } + + private static class CalculatorImpl implements Calculator { + private final DeployLogger log; + private final ApplicationPackage appPkg; + + private final ConcurrentMap<String, Long> modelCost = new ConcurrentHashMap<>(); + + private CalculatorImpl(ApplicationPackage appPkg, DeployLogger log) { + this.appPkg = appPkg; + this.log = log; + } + + @Override + public long aggregatedModelCostInBytes() { + return modelCost.values().stream().mapToLong(Long::longValue).sum(); + } + + @Override + public void registerModel(ApplicationFile f) { + String path = f.getPath().getRelative(); + if (alreadyAnalyzed(path)) return; + log.log(Level.FINE, () -> "Register model '%s'".formatted(path)); + if (f.exists() && appPkg != null) { + var memoryStats = OnnxModelProbe.probeMemoryStats(appPkg, f.getPath()).orElse(null); + if (memoryStats != null) { + log.log(Level.FINE, () -> "Register model '%s' with memory stats: %s".formatted(path, memoryStats)); + deductJvmHeapSizeWithModelCost(f.getSize(), memoryStats, path); + } else { + deductJvmHeapSizeWithModelCost(f.getSize(), path); + } + } else { + deductJvmHeapSizeWithModelCost(0, path); + } + } + + @Override + public void registerModel(ModelReference ref) { + log.log(Level.FINE, () -> "Register model '%s'".formatted(ref.toString())); + if (ref.path().isPresent()) { + var path = Paths.get(ref.path().get().value()); + var source = path.getFileName().toString(); + if (alreadyAnalyzed(source)) return; + deductJvmHeapSizeWithModelCost(uncheck(() -> Files.exists(path) ? Files.size(path) : 0), source); + } else if (ref.url().isPresent()) deductJvmHeapSizeWithModelCost(URI.create(ref.url().get().value())); + else throw new IllegalStateException(ref.toString()); + } + + private void deductJvmHeapSizeWithModelCost(URI uri) { + if (alreadyAnalyzed(uri.toString())) return; + if (uri.getScheme().equals("http") || uri.getScheme().equals("https")) { + try { + var timeout = Duration.ofSeconds(3); + var httpClient = HttpClient.newBuilder().connectTimeout(timeout).build(); + var request = HttpRequest.newBuilder(uri).timeout(timeout).method("HEAD", HttpRequest.BodyPublishers.noBody()).build(); + var response = httpClient.send(request, HttpResponse.BodyHandlers.discarding()); + var contentLength = response.headers().firstValue("Content-Length").orElse("0"); + log.log(Level.FINE, () -> "Got content length '%s' for '%s'".formatted(contentLength, uri)); + deductJvmHeapSizeWithModelCost(Long.parseLong(contentLength), uri.toString()); + } catch (IllegalArgumentException | InterruptedException | IOException e) { + log.log(Level.INFO, () -> "Failed to get model size for '%s': %s".formatted(uri, e.getMessage()), e); + } + } + } + + private void deductJvmHeapSizeWithModelCost(long size, String source) { + long fallbackModelSize = 1024*1024*1024; + long estimatedCost = Math.max(300*1024*1024, (long) (1.4D * (size > 0 ? size : fallbackModelSize) + 100*1024*1024)); + log.log(Level.FINE, () -> + "Estimated %s footprint for model of size %s ('%s')".formatted(mb(estimatedCost), mb(size), source)); + modelCost.put(source, estimatedCost); + } + + private void deductJvmHeapSizeWithModelCost(long size, OnnxModelProbe.MemoryStats stats, String source) { + long estimatedCost = (long)(1.1D * stats.vmSize()); + log.log(Level.FINE, () -> + "Estimated %s footprint for model of size %s ('%s')".formatted(mb(estimatedCost), mb(size), source)); + modelCost.put(source, estimatedCost); + } + + private boolean alreadyAnalyzed(String source) { return modelCost.containsKey(source); } + + private static String mb(long bytes) { return "%dMB".formatted(bytes / (1024*1024)); } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 28ff8dff620..727a18aee2c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -21,6 +21,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; @@ -197,7 +198,8 @@ public class VespaModelFactory implements ModelFactory { .zone(zone) .now(clock.instant()) .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()) - .wantedDockerImageRepo(modelContext.wantedDockerImageRepo()); + .wantedDockerImageRepo(modelContext.wantedDockerImageRepo()) + .onnxModelCost(modelContext.properties().hostedVespa() ? new DefaultOnnxModelCost() : OnnxModelCost.disabled()); modelContext.previousModel().ifPresent(builder::previousModel); modelContext.reindexing().ifPresent(builder::reindexing); return builder.build(validationParameters); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java new file mode 100644 index 00000000000..9e231239521 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; + +import java.util.logging.Level; + +/** + * Validates that the container node flavour has enough resources to run configured ONNX models. + * + * @author bjorncs + */ +public class JvmHeapSizeValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState ds) { + if (!ds.featureFlags().dynamicHeapSize()) return; + if (!ds.isHostedTenantApplication(model.getAdmin().getApplicationType())) return; + + model.getContainerClusters().forEach((clusterId, appCluster) -> { + var mp = appCluster.getMemoryPercentage().orElse(null); + if (mp == null) return; + if (mp.availableMemoryGb().isEmpty()) { + ds.getDeployLogger().log(Level.FINE, "Host resources unknown or percentage overridden with 'allocated-memory'"); + return; + } + long jvmModelCost = appCluster.onnxModelCost().aggregatedModelCostInBytes(); + if (jvmModelCost > 0) { + int percentLimit = 15; + if (mp.percentage() < percentLimit) { + throw new IllegalArgumentException( + ("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") + .formatted(clusterId, mp.percentage(), percentLimit, jvmModelCost / (1024D * 1024 * 1024))); + } + double gbLimit = 0.6; + double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); + if (availableMemoryGb < gbLimit) { + throw new IllegalArgumentException( + ("Allocated memory to JVM in cluster '%s' is too low (%.2fGB < %.2fGB). " + + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") + .formatted(clusterId, availableMemoryGb, gbLimit, jvmModelCost / (1024D * 1024 * 1024))); + } + } + }); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java new file mode 100644 index 00000000000..d9dd3729bd3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; + +/** + * Validates that config using s3:// urls is used in public system and with nodes that are exclusive. + * + * @author hmusum + */ +public class UrlConfigValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState state) { + if (! state.isHostedTenantApplication(model.getAdmin().getApplicationType())) return; + + model.getContainerClusters().forEach((__, cluster) -> { + var isExclusive = hasExclusiveNodes(model, cluster); + validateS3UlsInConfig(state, cluster, isExclusive); + }); + } + + private static boolean hasExclusiveNodes(VespaModel model, ApplicationContainerCluster cluster) { + return model.hostSystem().getHosts() + .stream() + .flatMap(hostResource -> hostResource.spec().membership().stream()) + .filter(membership -> membership.cluster().id().equals(cluster.id())) + .anyMatch(membership -> membership.cluster().isExclusive()); + } + + private static void validateS3UlsInConfig(DeployState state, ApplicationContainerCluster cluster, boolean isExclusive) { + if (hasS3UrlInConfig(cluster)) { + // TODO: Would be even better if we could add which config/field the url is set for in the error message + String message = "Found s3:// urls in config for container cluster " + cluster.getName(); + if ( ! state.zone().system().isPublic()) + throw new IllegalArgumentException(message + ". This is only supported in public systems"); + else if ( ! isExclusive) + throw new IllegalArgumentException(message + ". Nodes in the cluster need to be 'exclusive'," + + " see https://cloud.vespa.ai/en/reference/services#nodes"); + } + } + + private static boolean hasS3UrlInConfig(ApplicationContainerCluster cluster) { + return cluster.userConfiguredUrls().all().stream() + .anyMatch(url -> url.startsWith("s3://")); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 53a553ee624..30aafe67be7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -87,6 +87,8 @@ public class Validation { new AccessControlFilterExcludeValidator().validate(model, deployState); new CloudUserFilterValidator().validate(model, deployState); new CloudHttpConnectorValidator().validate(model, deployState); + new UrlConfigValidator().validate(model, deployState); + new JvmHeapSizeValidator().validate(model, deployState); additionalValidators.forEach(v -> v.validate(model, deployState)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java index bbfa939f8a3..f265f2d09a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java @@ -7,6 +7,7 @@ import com.yahoo.schema.document.Matching; import com.yahoo.schema.document.MatchType; import com.yahoo.schema.document.NormalizeLevel; import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.processing.NGramMatch; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -89,7 +90,7 @@ public class IndexingScriptChangeMessageBuilder { MatchType type = matching.getType(); String retval = type.getName(); if (type == MatchType.GRAM) { - retval += " (size " + matching.getGramSize() + ")"; + retval += " (size " + matching.getGramSize().orElse(NGramMatch.DEFAULT_GRAM_SIZE) + ")"; } return retval; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java index 7501f6162c7..9ecd359f90d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java @@ -7,11 +7,12 @@ import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; -import com.yahoo.vespa.model.container.component.HuggingFaceEmbedder; -import com.yahoo.vespa.model.container.component.HuggingFaceTokenizer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.BertEmbedder; import com.yahoo.vespa.model.container.component.ColBertEmbedder; import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.HuggingFaceEmbedder; +import com.yahoo.vespa.model.container.component.HuggingFaceTokenizer; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; @@ -35,19 +36,20 @@ public class DomComponentBuilder extends VespaDomBuilder.DomConfigProducerBuilde @Override protected Component<? super Component<?, ?>, ?> doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element spec) { - var component = buildComponent(spec, deployState); + var component = buildComponent(spec, deployState, ancestor); addChildren(deployState, ancestor, spec, component); return component; } - private Component<? super Component<?, ?>, ?> buildComponent(Element spec, DeployState state) { + private Component<? super Component<?, ?>, ?> buildComponent( + Element spec, DeployState state, TreeConfigProducer<AnyConfigProducer> ancestor) { if (spec.hasAttribute("type")) { var type = spec.getAttribute("type"); return switch (type) { - case "hugging-face-embedder" -> new HuggingFaceEmbedder(spec, state); + case "hugging-face-embedder" -> new HuggingFaceEmbedder((ApplicationContainerCluster)ancestor, spec, state); case "hugging-face-tokenizer" -> new HuggingFaceTokenizer(spec, state); - case "bert-embedder" -> new BertEmbedder(spec, state); - case "colbert-embedder" -> new ColBertEmbedder(spec, state); + case "colbert-embedder" -> new ColBertEmbedder((ApplicationContainerCluster)ancestor, spec, state); + case "bert-embedder" -> new BertEmbedder((ApplicationContainerCluster)ancestor, spec, state); default -> throw new IllegalArgumentException("Unknown component type '%s'".formatted(type)); }; } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index b9021912244..4e97b20a3a9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -8,14 +8,14 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ApplicationClusterInfo; -import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.config.ApplicationBundlesConfig; @@ -40,13 +40,16 @@ import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.filedistribution.UserConfiguredFiles; +import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.stream.Collectors; import static com.yahoo.vespa.model.container.docproc.DocprocChains.DOCUMENT_TYPE_MANAGER_CLASS; @@ -82,6 +85,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); private final Set<String> previousHosts; + private final OnnxModelCost.Calculator onnxModelCost; + private final DeployLogger logger; private ContainerModelEvaluation modelEvaluation; @@ -92,6 +97,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private int zookeeperSessionTimeoutSeconds = 30; private final int transport_events_before_wakeup; private final int transport_connections_per_target; + private final boolean dynamicHeapSize; /** The heap size % of total memory available to the JVM process. */ private final int heapSizePercentageOfAvailableMemory; @@ -100,9 +106,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private List<ApplicationClusterEndpoint> endpoints = List.of(); + private UserConfiguredUrls userConfiguredUrls = new UserConfiguredUrls(); + public ApplicationContainerCluster(TreeConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { super(parent, configSubId, clusterId, deployState, true, 10); this.tlsClientAuthority = deployState.tlsClientAuthority(); + dynamicHeapSize = deployState.featureFlags().dynamicHeapSize(); previousHosts = Collections.unmodifiableSet(deployState.getPreviousModel().stream() .map(Model::allocatedHosts) .map(AllocatedHosts::getHosts) @@ -125,8 +134,13 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat heapSizePercentageOfAvailableMemory = deployState.featureFlags().heapSizePercentage() > 0 ? Math.min(99, deployState.featureFlags().heapSizePercentage()) : defaultHeapSizePercentageOfAvailableMemory; + onnxModelCost = deployState.onnxModelCost().newCalculator( + deployState.getApplicationPackage(), deployState.getDeployLogger()); + logger = deployState.getDeployLogger(); } + public UserConfiguredUrls userConfiguredUrls() { return userConfiguredUrls; } + @Override protected void doPrepare(DeployState deployState) { super.doPrepare(deployState); @@ -147,7 +161,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat if (containers.isEmpty()) return; // Files referenced from user configs to all components. - UserConfiguredFiles files = new UserConfiguredFiles(deployState.getFileRegistry(), deployState.getDeployLogger()); + UserConfiguredFiles files = new UserConfiguredFiles(deployState.getFileRegistry(), + deployState.getDeployLogger(), + userConfiguredUrls); for (Component<?, ?> component : getAllComponents()) { files.register(component); } @@ -182,19 +198,25 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public void setMemoryPercentage(Integer memoryPercentage) { this.memoryPercentage = memoryPercentage; } @Override - public Optional<Integer> getMemoryPercentage() { - if (memoryPercentage != null) return Optional.of(memoryPercentage); + public Optional<JvmMemoryPercentage> getMemoryPercentage() { + if (memoryPercentage != null) return Optional.of(JvmMemoryPercentage.of(memoryPercentage)); if (isHostedVespa()) { int availableMemoryPercentage = getHostClusterId().isPresent() ? heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster : heapSizePercentageOfAvailableMemory; - if (getContainers().isEmpty()) return Optional.of(availableMemoryPercentage); // Node memory is not known + if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(availableMemoryPercentage)); // Node memory is not known // Node memory is known so convert available memory percentage to node memory percentage - double totalMemory = getContainers().get(0).getHostResource().realResources().memoryGb(); - double availableMemory = totalMemory - Host.memoryOverheadGb; - return Optional.of((int) (availableMemory / totalMemory * availableMemoryPercentage)); + double totalMemory = dynamicHeapSize + ? getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow() + : getContainers().get(0).getHostResource().realResources().memoryGb(); + double jvmHeapDeductionGb = dynamicHeapSize ? onnxModelCost.aggregatedModelCostInBytes() / (1024D * 1024 * 1024) : 0; + double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb); + int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); + logger.log(Level.FINE, () -> "memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" + .formatted(memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); + return Optional.of(JvmMemoryPercentage.of(memoryPercentage, availableMemory)); } return Optional.empty(); } @@ -203,49 +225,23 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private void createEndpoints(DeployState deployState) { if (!deployState.isHosted()) return; if (deployState.getProperties().applicationId().instance().isTester()) return; + // Add endpoints provided by the controller + List<String> hosts = getContainers().stream().map(AbstractService::getHostName).sorted().toList(); List<ApplicationClusterEndpoint> endpoints = new ArrayList<>(); - - List<String> hosts = getContainers().stream() - .map(AbstractService::getHostName) - .sorted() - .toList(); - - Set<ContainerEndpoint> endpointsFromController = deployState.getEndpoints(); - // Add zone-scoped endpoints if not provided by the controller - // TODO(mpolden): Remove this when controller always includes zone-scope endpoints, and config models < 8.230 are gone - if (endpointsFromController.stream().noneMatch(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.zone)) { - for (String suffix : deployState.getProperties().zoneDnsSuffixes()) { - ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom( - deployState.zone().system(), - ClusterSpec.Id.from(getName()), - deployState.getProperties().applicationId(), - suffix); - endpoints.add(ApplicationClusterEndpoint.builder() - .zoneScope() - .sharedL4Routing() - .dnsName(l4Name) - .hosts(hosts) - .clusterId(getName()) - .authMethod(ApplicationClusterEndpoint.AuthMethod.mtls) - .build()); - } - } - - // Include all endpoints provided by controller - endpointsFromController.stream() - .filter(ce -> ce.clusterId().equals(getName())) - .forEach(ce -> ce.names().forEach( - name -> endpoints.add(ApplicationClusterEndpoint.builder() - .scope(ce.scope()) - .weight(ce.weight().orElse(1)) // Default to weight=1 if not set - .routingMethod(ce.routingMethod()) - .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) - .hosts(hosts) - .clusterId(getName()) - .authMethod(ce.authMethod()) - .build()) - )); - this.endpoints = List.copyOf(endpoints); + deployState.getEndpoints().stream() + .filter(ce -> ce.clusterId().equals(getName())) + .forEach(ce -> ce.names().forEach( + name -> endpoints.add(ApplicationClusterEndpoint.builder() + .scope(ce.scope()) + .weight(ce.weight().orElse(1)) + .routingMethod(ce.routingMethod()) + .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) + .hosts(hosts) + .clusterId(getName()) + .authMethod(ce.authMethod()) + .build()) + )); + this.endpoints = Collections.unmodifiableList(endpoints); } @Override @@ -299,12 +295,15 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat @Override public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); + var memoryPct = getMemoryPercentage().orElse(null); + int heapsize = memoryPct != null && memoryPct.availableMemoryGb().isPresent() + ? (int) (memoryPct.availableMemoryGb().getAsDouble() * 1024) : 1536; builder.jvm.verbosegc(true) .availableProcessors(0) .compressedClassSpaceSize(0) - .minHeapsize(1536) - .heapsize(1536); - getMemoryPercentage().ifPresent(percentage -> builder.jvm.heapSizeAsPercentageOfPhysicalMemory(percentage)); + .minHeapsize(heapsize) + .heapsize(heapsize); + if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.percentage()); } @Override @@ -373,6 +372,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat @Override public String name() { return getName(); } + public OnnxModelCost.Calculator onnxModelCost() { return onnxModelCost; } + public static class MbusParams { // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) final Double maxConcurrentFactor; @@ -390,4 +391,14 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } } + public static class UserConfiguredUrls { + + private final Set<String> urls = new HashSet<>(); + + public void add(String url) { urls.add(url); } + + public Set<String> all() { return urls; } + + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6bbc24e8739..fa13e7ec9d6 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -62,6 +62,7 @@ import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.search.SearchCluster; + import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; @@ -71,6 +72,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalDouble; import java.util.Set; import java.util.TreeSet; @@ -718,5 +720,11 @@ public abstract class ContainerCluster<CONTAINER extends Container> * Returns the percentage of host physical memory this application has specified for nodes in this cluster, * or empty if this is not specified by the application. */ - public Optional<Integer> getMemoryPercentage() { return Optional.empty(); } + public record JvmMemoryPercentage(int percentage, OptionalDouble availableMemoryGb) { + static JvmMemoryPercentage of(int percentage) { return new JvmMemoryPercentage(percentage, OptionalDouble.empty()); } + static JvmMemoryPercentage of(int percentage, double availableMemoryGb) { + return new JvmMemoryPercentage(percentage, OptionalDouble.of(availableMemoryGb)); + } + } + public Optional<JvmMemoryPercentage> getMemoryPercentage() { return Optional.empty(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 906ef739ef1..1b47f59653e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -45,10 +45,6 @@ public class ContainerModelEvaluation implements private final RankProfileList rankProfileList; private final FileDistributedOnnxModels onnxModels; // For cluster specific ONNX model settings - public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList) { - this(cluster, rankProfileList, null); - } - public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList, FileDistributedOnnxModels onnxModels) { this.rankProfileList = Objects.requireNonNull(rankProfileList, "rankProfileList cannot be null"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java index 205848e1b67..76bb1a9e02a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java @@ -5,6 +5,7 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.embedding.BertBaseEmbedderConfig; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.xml.ModelIdResolver; import org.w3c.dom.Element; @@ -33,7 +34,7 @@ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConf private final Integer onnxGpuDevice; - public BertEmbedder(Element xml, DeployState state) { + public BertEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.BertBaseEmbedder", INTEGRATION_BUNDLE_NAME, xml); model = ModelIdResolver.resolveToModelReference(getChild(xml, "transformer-model"), state); vocab = ModelIdResolver.resolveToModelReference(getChild(xml, "tokenizer-vocab"), state); @@ -49,6 +50,7 @@ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConf onnxInteropThreads = getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt).orElse(null); onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); + cluster.onnxModelCost().registerModel(model); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java index c0fdfe3dc64..63096ebcbe2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java @@ -5,7 +5,7 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.embedding.ColBertEmbedderConfig; -import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.xml.ModelIdResolver; import org.w3c.dom.Element; @@ -40,7 +40,7 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo private final Integer onnxIntraopThreads; private final Integer onnxGpuDevice; - public ColBertEmbedder(Element xml, DeployState state) { + public ColBertEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.ColBertEmbedder", INTEGRATION_BUNDLE_NAME, xml); var transformerModelElem = getOptionalChild(xml, "transformer-model").orElseThrow(); model = ModelIdResolver.resolveToModelReference(transformerModelElem, state); @@ -60,7 +60,7 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo onnxInteropThreads = getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt).orElse(null); onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); - + cluster.onnxModelCost().registerModel(model); } private static ModelReference resolveDefaultVocab(Element model, DeployState state) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java index f4017339699..41b80bf1cb2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java @@ -5,6 +5,7 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.xml.ModelIdResolver; import org.w3c.dom.Element; @@ -33,7 +34,7 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm private final Integer onnxGpuDevice; private final String poolingStrategy; - public HuggingFaceEmbedder(Element xml, DeployState state) { + public HuggingFaceEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.huggingface.HuggingFaceEmbedder", INTEGRATION_BUNDLE_NAME, xml); var transformerModelElem = getOptionalChild(xml, "transformer-model").orElseThrow(); model = ModelIdResolver.resolveToModelReference(transformerModelElem, state); @@ -51,6 +52,7 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); poolingStrategy = getChildValue(xml, "pooling-strategy").orElse(null); + cluster.onnxModelCost().registerModel(model); } private static ModelReference resolveDefaultVocab(Element model, DeployState state) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index f0296d49472..3261d454b4f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; @@ -56,12 +57,14 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> private QueryProfiles queryProfiles; private SemanticRules semanticRules; private PageTemplates pageTemplates; + private ApplicationPackage app; public ContainerSearch(DeployState deployState, ApplicationContainerCluster cluster, SearchChains chains) { super(chains); this.globalPhase = deployState.featureFlags().enableGlobalPhase(); this.useReconfigurableDispatcher = deployState.featureFlags().useReconfigurableDispatcher(); this.schemasWithGlobalPhase = getSchemasWithGlobalPhase(deployState); + this.app = deployState.getApplicationPackage(); this.owningCluster = cluster; owningCluster.addComponent(Component.fromClassAndBundle(CompiledQueryProfileRegistry.class, SEARCH_AND_DOCPROC_BUNDLE)); @@ -96,6 +99,9 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> if ( ! schemasWithGlobalPhase.contains(documentDb.getSchemaName())) continue; var factory = new RankProfilesEvaluatorComponent(documentDb); if ( ! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { + var onnxModels = documentDb.getDerivedConfiguration().getRankProfileList().getOnnxModels(); + onnxModels.asMap().forEach( + (__, model) -> owningCluster.onnxModelCost().registerModel(app.getFile(model.getFilePath()))); owningCluster.addComponent(factory); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 35b0213bf59..d9c4dea478c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -778,6 +778,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { !container.getHostResource().realResources().gpuResources().isZero()); onnxModel.setGpuDevice(gpuDevice, hasGpu); } + cluster.onnxModelCost().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath())); } cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles, models)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index bb72eda7d04..d18309ef0af 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -256,7 +256,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem for (ContainerModel containerModel : containers) { Optional<String> hostClusterId = containerModel.getCluster().getHostClusterId(); if (hostClusterId.isPresent() && hostClusterId.get().equals(clusterId) && containerModel.getCluster().getMemoryPercentage().isPresent()) { - return containerModel.getCluster().getMemoryPercentage().get() * 0.01; + return containerModel.getCluster().getMemoryPercentage().get().percentage() * 0.01; } } return 0.0; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java index 8bed5e64bf5..03541ecadf3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java @@ -11,6 +11,7 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; + import com.yahoo.yolean.Exceptions; import java.io.File; @@ -21,6 +22,8 @@ import java.util.Map; import java.util.Optional; import java.util.logging.Level; +import static com.yahoo.vespa.model.container.ApplicationContainerCluster.UserConfiguredUrls; + /** * Utility methods for registering file distribution of files/paths/urls/models defined by the user. * @@ -30,10 +33,12 @@ public class UserConfiguredFiles implements Serializable { private final FileRegistry fileRegistry; private final DeployLogger logger; + private final UserConfiguredUrls userConfiguredUrls; - public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger) { + public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger, UserConfiguredUrls userConfiguredUrls) { this.fileRegistry = fileRegistry; this.logger = logger; + this.userConfiguredUrls = userConfiguredUrls; } /** @@ -133,7 +138,10 @@ public class UserConfiguredFiles implements Serializable { Path path; if (isModelType) { var modelReference = ModelReference.valueOf(builder.getValue()); - if (modelReference.path().isEmpty()) return; + if (modelReference.path().isEmpty()) { + modelReference.url().ifPresent(url -> userConfiguredUrls.add(url.value())); + return; + } path = Path.fromString(modelReference.path().get().value()); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java index 7c86267c1b6..38dda3e29ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java @@ -18,6 +18,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.Optional; + +import static com.yahoo.yolean.Exceptions.uncheck; /** * Defers to 'vespa-analyze-onnx-model' to determine the output type given @@ -29,6 +32,7 @@ import java.util.Map; public class OnnxModelProbe { private static final String binary = "vespa-analyze-onnx-model"; + private static final ObjectMapper jsonParser = new ObjectMapper(); static TensorType probeModel(ApplicationPackage app, Path modelPath, String outputName, Map<String, TensorType> inputTypes) { TensorType outputType = TensorType.empty; @@ -41,8 +45,9 @@ public class OnnxModelProbe { // Otherwise, run vespa-analyze-onnx-model if the model is available if (outputType.equals(TensorType.empty) && app.getFile(modelPath).exists()) { String jsonInput = createJsonInput(app.getFileReference(modelPath).getAbsolutePath(), inputTypes); - String jsonOutput = callVespaAnalyzeOnnxModel(jsonInput); + var jsonOutput = callVespaAnalyzeOnnxModel(jsonInput); outputType = outputTypeFromJson(jsonOutput, outputName); + writeMemoryStats(app, modelPath, MemoryStats.fromJson(jsonOutput)); if ( ! outputType.equals(TensorType.empty)) { writeProbedOutputType(app, modelPath, contextKey, outputType); } @@ -53,6 +58,22 @@ public class OnnxModelProbe { return outputType; } + public static Optional<MemoryStats> probeMemoryStats(ApplicationPackage app, Path modelPath) { + return Optional.of(app.getFile(memoryStatsPath(modelPath))) + .filter(ApplicationFile::exists) + .map(file -> MemoryStats.fromJson(uncheck(() -> jsonParser.readTree(file.createReader())))); + } + + private static void writeMemoryStats(ApplicationPackage app, Path modelPath, MemoryStats memoryStats) throws IOException { + String path = app.getFileReference(memoryStatsPath(modelPath)).getAbsolutePath(); + IOUtils.writeFile(path, memoryStats.toJson().toPrettyString(), false); + } + + private static Path memoryStatsPath(Path modelPath) { + var fileName = OnnxModelInfo.asValidIdentifier(modelPath.getRelative()) + ".memory_stats"; + return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(fileName); + } + private static String createContextKey(String onnxName, Map<String, TensorType> inputTypes) { StringBuilder key = new StringBuilder().append(onnxName).append(":"); inputTypes.entrySet().stream().sorted(Map.Entry.comparingByKey()) @@ -95,9 +116,7 @@ public class OnnxModelProbe { return TensorType.empty; } - private static TensorType outputTypeFromJson(String json, String outputName) throws IOException { - ObjectMapper m = new ObjectMapper(); - JsonNode root = m.readTree(json); + private static TensorType outputTypeFromJson(JsonNode root, String outputName) throws IOException { if ( ! root.isObject() || ! root.has("outputs")) { return TensorType.empty; } @@ -123,7 +142,7 @@ public class OnnxModelProbe { return out.toString(); } - private static String callVespaAnalyzeOnnxModel(String jsonInput) throws IOException, InterruptedException { + private static JsonNode callVespaAnalyzeOnnxModel(String jsonInput) throws IOException, InterruptedException { StringBuilder output = new StringBuilder(); ProcessBuilder processBuilder = new ProcessBuilder(binary, "--probe-types"); @@ -148,7 +167,16 @@ public class OnnxModelProbe { throw new IllegalArgumentException("Error from '" + binary + "'. Return code: " + returnCode + ". " + "Output: '" + output + "'"); } - return output.toString(); + return jsonParser.readTree(output.toString()); + } + + public record MemoryStats(long vmSize, long vmRss) { + static MemoryStats fromJson(JsonNode json) { + return new MemoryStats(json.get("vm_size").asLong(), json.get("vm_rss").asLong()); + } + JsonNode toJson() { + return jsonParser.createObjectNode().put("vm_size", vmSize).put("vm_rss", vmRss); + } } } diff --git a/config-model/src/test/derived/ngram/chunk.sd b/config-model/src/test/derived/ngram/chunk.sd new file mode 100644 index 00000000000..7c2a7465327 --- /dev/null +++ b/config-model/src/test/derived/ngram/chunk.sd @@ -0,0 +1,20 @@ +schema chunk { + + document chunk { + field content type string { + indexing: summary | index + match { + gram + gram-size: 3 + } + } + } + + document-summary content-summary inherits default { + summary content_dynamic type string { + source: content + dynamic + } + } + +} diff --git a/config-model/src/test/derived/ngram/index-info.cfg b/config-model/src/test/derived/ngram/index-info.cfg new file mode 100644 index 00000000000..72b6760ceb5 --- /dev/null +++ b/config-model/src/test/derived/ngram/index-info.cfg @@ -0,0 +1,21 @@ +indexinfo[].name "chunk" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "string" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "ngram 3" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "string" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "ngram 3" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "dynteaser" diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 2f8a8bddf20..38f51323ee2 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -148,7 +148,7 @@ public class ModelProvisioningTest { assertEquals("-Xlog:gc", mydisc2.getContainers().get(1).getJvmOptions()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(0).getPreLoad()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(1).getPreLoad()); - assertEquals(Optional.of(45), mydisc2.getMemoryPercentage()); + assertEquals(45, mydisc2.getMemoryPercentage().get().percentage()); assertEquals(Optional.of("-XX:+UseParNewGC"), mydisc2.getJvmGCOptions()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); mydisc2.getConfig(qrStartBuilder); @@ -288,10 +288,11 @@ public class ModelProvisioningTest { assertEquals(2025077080L, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); - assertEquals(1, logger.msgs().size()); + var msgs = logger.msgs().stream().filter(m -> m.level().equals(Level.WARNING)).toList(); + assertEquals(1, msgs.size()); assertEquals("Declaring combined cluster with <nodes of=\"...\"> is deprecated without replacement, " + "and the feature will be removed in Vespa 9. Use separate container and content clusters instead", - logger.msgs().get(0).message); + msgs.get(0).message); } @Test diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java new file mode 100644 index 00000000000..4481445858a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java @@ -0,0 +1,15 @@ +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +public class NGramTestCase extends AbstractExportingTestCase { + + @Test + void testNGram() throws IOException, ParseException { + assertCorrectDeriving("ngram"); + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java index 06ea202b9c3..551542a84ba 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java @@ -27,15 +27,15 @@ public class NGramTestCase extends AbstractSchemaTestCase { SDField gram1 = schema.getConcreteField("gram_1"); assertEquals(MatchType.GRAM, gram1.getMatching().getType()); - assertEquals(1, gram1.getMatching().getGramSize()); + assertEquals(1, gram1.getMatching().getGramSize().getAsInt()); SDField gram2 = schema.getConcreteField("gram_2"); assertEquals(MatchType.GRAM, gram2.getMatching().getType()); - assertEquals(-1, gram2.getMatching().getGramSize()); // Not set explicitly + assertTrue(gram2.getMatching().getGramSize().isEmpty()); SDField gram3 = schema.getConcreteField("gram_3"); assertEquals(MatchType.GRAM, gram3.getMatching().getType()); - assertEquals(3, gram3.getMatching().getGramSize()); + assertEquals(3, gram3.getMatching().getGramSize().getAsInt()); assertEquals("input gram_1 | ngram 1 | index gram_1 | summary gram_1", gram1.getIndexingScript().iterator().next().toString()); assertEquals("input gram_2 | ngram 2 | attribute gram_2 | index gram_2", gram2.getIndexingScript().iterator().next().toString()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java new file mode 100644 index 00000000000..447614b8396 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -0,0 +1,127 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.model.VespaModel; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author bjorncs + */ +class JvmHeapSizeValidatorTest { + + @Test + void fails_on_too_low_jvm_percentage() throws IOException, SAXException { + var deployState = createDeployState(8, 7L * 1024 * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); + String expectedMessage = "Allocated percentage of memory of JVM in cluster 'container' is too low (3% < 15%). Estimated cost of ONNX models is 7.00GB"; + assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); + } + + @Test + void fails_on_too_low_heap_size() throws IOException, SAXException { + var deployState = createDeployState(2.2, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); + String expectedMessage = "Allocated memory to JVM in cluster 'container' is too low (0.50GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."; + assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); + } + + @Test + void accepts_adequate_heap_size() throws IOException, SAXException { + var deployState = createDeployState(8, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + assertDoesNotThrow(() -> new JvmHeapSizeValidator().validate(model, deployState)); + } + + @Test + void accepts_services_with_explicit_jvm_size() throws IOException, SAXException { + String servicesXml = + """ + <?xml version="1.0" encoding="utf-8" ?> + <services version='1.0'> + <container version='1.0'> + <nodes count="2"> + <jvm allocated-memory='5%'/> + <resources vcpu="4" memory="2Gb" disk="125Gb"/> + </nodes> + <component id="hf-embedder" type="hugging-face-embedder"> + <transformer-model url="https://my/url/model.onnx"/> + <tokenizer-model path="app/tokenizer.json"/> + </component> + </container> + </services>"""; + var deployState = createDeployState(servicesXml, 2, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + assertDoesNotThrow(() -> new JvmHeapSizeValidator().validate(model, deployState)); + } + + private static DeployState createDeployState(String servicesXml, double nodeGb, long modelCostBytes) { + return new DeployState.Builder() + .applicationPackage( + new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build()) + .modelHostProvisioner(new InMemoryProvisioner(5, new NodeResources(4, nodeGb, 125, 0.3), true)) + .properties(new TestProperties().setHostedVespa(true).setDynamicHeapSize(true)) + .onnxModelCost(new ModelCostDummy(modelCostBytes)) + .build(); + } + + private static DeployState createDeployState(double nodeGb, long modelCostBytes) { + String servicesXml = + """ + <?xml version="1.0" encoding="utf-8" ?> + <services version='1.0'> + <container version='1.0'> + <nodes count="2"> + <resources vcpu="4" memory="%fGb" disk="125Gb"/> + </nodes> + <component id="hf-embedder" type="hugging-face-embedder"> + <transformer-model url="https://my/url/model.onnx"/> + <tokenizer-model path="app/tokenizer.json"/> + </component> + </container> + </services>""".formatted(nodeGb); + return createDeployState(servicesXml, nodeGb, modelCostBytes); + } + + private static class ModelCostDummy implements OnnxModelCost, OnnxModelCost.Calculator { + final AtomicLong totalCost = new AtomicLong(); + final long modelCost; + + ModelCostDummy(long modelCost) { this.modelCost = modelCost; } + + @Override public Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger) { return this; } + @Override public long aggregatedModelCostInBytes() { return totalCost.get(); } + @Override public void registerModel(ApplicationFile path) {} + + @Override + public void registerModel(ModelReference ref) { + assertEquals("https://my/url/model.onnx", ref.url().orElseThrow().value().toString()); + totalCost.addAndGet(modelCost); + } + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java new file mode 100644 index 00000000000..cef4d8c27dd --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java @@ -0,0 +1,107 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.embedding.BertBaseEmbedderConfig; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.model.VespaModel; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static com.yahoo.config.provision.Environment.prod; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class UrlConfigValidatorTest { + + @Test + void failsWhenContainerNodesNotExclusive() throws IOException, SAXException { + runValidatorOnApp(true, SystemName.Public); // Exclusive nodes in public => success + + assertEquals("Found s3:// urls in config for container cluster default. This is only supported in public systems", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(false, SystemName.main)) + .getMessage()); + + assertEquals("Found s3:// urls in config for container cluster default. This is only supported in public systems", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(true, SystemName.main)) + .getMessage()); + + assertEquals("Found s3:// urls in config for container cluster default. Nodes in the cluster need to be 'exclusive'," + + " see https://cloud.vespa.ai/en/reference/services#nodes", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(false, SystemName.Public)) + .getMessage()); + } + + private static String containerXml(boolean isExclusive) { + return """ + <container version='1.0' id='default'> + <component id='transformer' class='ai.vespa.embedding.BertBaseEmbedder' bundle='model-integration'> + <config name='embedding.bert-base-embedder'> + <transformerModel url='s3://models/minilm-l6-v2/sentence_all_MiniLM_L6_v2.onnx' path='foo'/> + <tokenizerVocab url='s3://models/bert-base-uncased.txt'/> + </config> + </component> + <search/> + <document-api/> + <nodes count='2' exclusive='%s' /> + </container> + """.formatted(Boolean.toString(isExclusive)); + } + + private static void runValidatorOnApp(boolean isExclusive, SystemName systemName) throws IOException, SAXException { + String container = containerXml(isExclusive); + String servicesXml = """ + <services version='1.0'> + %s + </services> + """.formatted(container); + ApplicationPackage app = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build(); + DeployState deployState = createDeployState(app, systemName); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + new UrlConfigValidator().validate(model, deployState); + } + + private static DeployState createDeployState(ApplicationPackage app, SystemName systemName) { + boolean isHosted = true; + var builder = new DeployState.Builder() + .applicationPackage(app) + .zone(new Zone(systemName, prod, RegionName.from("us-east-3"))) + .properties(new TestProperties().setHostedVespa(isHosted)) + .fileRegistry(new MockFileRegistry()); + + Map<ConfigDefinitionKey, ConfigDefinition> defs = new HashMap<>(); + defs.put(new ConfigDefinitionKey(BertBaseEmbedderConfig.CONFIG_DEF_NAME, BertBaseEmbedderConfig.CONFIG_DEF_NAMESPACE), + new ConfigDefinition(BertBaseEmbedderConfig.CONFIG_DEF_NAME, BertBaseEmbedderConfig.CONFIG_DEF_SCHEMA)); + builder.configDefinitionRepo(new ConfigDefinitionRepo() { + @Override + public Map<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> getConfigDefinitions() { + return defs; + } + + @Override + public com.yahoo.vespa.config.buildergen.ConfigDefinition get(ConfigDefinitionKey key) { + return defs.get(key); + } + }); + return builder.build(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 894fc55c014..3bdb60a0a8d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -42,14 +42,14 @@ import java.util.Objects; import java.util.OptionalInt; import java.util.Set; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.exclusive; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.shared; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.application; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.global; -import static com.yahoo.config.provision.SystemName.cd; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.zone; import static com.yahoo.config.provision.SystemName.main; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Simon Thoresen Hult @@ -365,62 +365,23 @@ public class ContainerClusterTest { @Test void generatesCorrectRoutingInfo() { - // main system: assertNames(main, ApplicationId.from("t1", "a1", "i1"), - Set.of(), + Set.of(new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.i1.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.i1.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(), - List.of("search-cluster.a1.t1.endpoint.suffix")); - - assertNames(main, - ApplicationId.from("t1", "default", "default"), - Set.of(), - List.of("search-cluster.default.t1.endpoint.suffix")); - - assertNames(main, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), + Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar")), + new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4)), + Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), + new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), + new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); - - // cd system: - assertNames(cd, - ApplicationId.from("t1", "a1", "i1"), - Set.of(), - List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(), - List.of("search-cluster.cd.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "default", "default"), - Set.of(), - List.of("search-cluster.cd.default.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), - List.of("search-cluster.cd.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", global, List.of("a.b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), - new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), - new ContainerEndpoint("not-supported", global, List.of("not.supported"), OptionalInt.empty(), exclusive)), - List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); - } private void assertNames(SystemName systemName, ApplicationId appId, Set<ContainerEndpoint> globalEndpoints, List<String> expectedSharedL4Names) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java index bb5ba840c2c..e2a25642575 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.model.SimpleConfigProducer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -68,7 +69,7 @@ public class UserConfiguredFilesTest { } private UserConfiguredFiles userConfiguredFiles() { - return new UserConfiguredFiles(fileRegistry, new BaseDeployLogger()); + return new UserConfiguredFiles(fileRegistry, new BaseDeployLogger(), new ApplicationContainerCluster.UserConfiguredUrls()); } @BeforeEach diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKeyWithTimestamp.java b/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKeyWithTimestamp.java new file mode 100644 index 00000000000..ecc1cf71113 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKeyWithTimestamp.java @@ -0,0 +1,39 @@ +package com.yahoo.config.provision; + +import com.yahoo.jdisc.Timer; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Random; + +/** + * @author gjoranv + */ +public record WireguardKeyWithTimestamp(WireguardKey key, Instant timestamp) { + + public static final int KEY_ROTATION_BASE = 60; + public static final int KEY_ROTATION_VARIANCE = 10; + public static final int KEY_EXPIRY = KEY_ROTATION_BASE + KEY_ROTATION_VARIANCE + 5; + + public WireguardKeyWithTimestamp { + if (key == null) throw new IllegalArgumentException("Wireguard key cannot be null"); + if (timestamp == null) timestamp = Instant.EPOCH; + } + + public static WireguardKeyWithTimestamp from(String key, long msTimestamp) { + return new WireguardKeyWithTimestamp(WireguardKey.from(key), Instant.ofEpochMilli(msTimestamp)); + } + + public boolean isDueForRotation(Timer timer, ChronoUnit unit, Random random) { + return timer.currentTime().isAfter(keyRotationDueAt(unit, random)); + } + + public boolean hasExpired(Timer timer, ChronoUnit unit) { + return timer.currentTime().isAfter(timestamp.plus(KEY_EXPIRY, unit)); + } + + private Instant keyRotationDueAt(ChronoUnit unit, Random random) { + return timestamp.plus(KEY_ROTATION_BASE + random.nextInt(KEY_ROTATION_VARIANCE), unit); + } + +} diff --git a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java index 3b24e1c1b8d..4f8d42e895b 100644 --- a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java +++ b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java @@ -111,7 +111,7 @@ public class FlagsHandlerTest { }, { "type": "blacklist", - "dimension": "application", + "dimension": "instance", "values": [ "app1", "app2" ] } ], @@ -127,7 +127,7 @@ public class FlagsHandlerTest { // GET on id2 should now return what was put verifySuccessfulRequest(Method.GET, "/data/" + FLAG2.id(), "", - "{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}"); + "{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}"); // The list of flag data should return id1 and id2 verifySuccessfulRequest(Method.GET, "/data", @@ -153,7 +153,7 @@ public class FlagsHandlerTest { // Get all recursivelly displays all flag data verifySuccessfulRequest(Method.GET, "/data?recursive=true", "", - "{\"flags\":[{\"id\":\"id1\",\"rules\":[{\"value\":false}]},{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}]}"); + "{\"flags\":[{\"id\":\"id1\",\"rules\":[{\"value\":false}]},{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}]}"); // Deleting both flags verifySuccessfulRequest(Method.DELETE, "/data/" + FLAG1.id(), "", ""); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index 154d2d0f2f0..042aa2423f3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -11,6 +11,7 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -35,15 +36,17 @@ public class DeployHandlerLogger implements DeployLogger { this.logroot = slime.setObject().setArray("log"); } + @Override public void log(Level level, String message) { log(level, () -> message); } + @Override public void log(Level level, Supplier<String> message) { log(level, message, null); } + @Override @SuppressWarnings("deprecation") - public void log(Level level, String message) { - if (level.intValue() <= LogLevel.DEBUG.intValue() && !verbose) - return; + public void log(Level level, Supplier<String> supplier, Throwable throwable) { + // Also tee to a normal log, Vespa log for example, but use level fine + log.log(Level.FINE, throwable, () -> prefix + supplier.get()); - logJson(level, message); - // Also tee to a normal log, Vespa log for example, but use level fine - log.log(Level.FINE, () -> prefix + message); + if (level.intValue() <= LogLevel.DEBUG.intValue() && !verbose) return; + logJson(level, supplier.get()); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 142f98e13e3..3e33b345437 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -201,6 +201,7 @@ public class ModelContextImpl implements ModelContext { private final boolean enableNestedMultivalueGrouping; private final boolean useReconfigurableDispatcher; private final int contentLayerMetadataFeatureLevel; + private final boolean dynamicHeapSize; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -243,6 +244,7 @@ public class ModelContextImpl implements ModelContext { this.enableNestedMultivalueGrouping = flagValue(source, appId, version, Flags.ENABLE_NESTED_MULTIVALUE_GROUPING); this.useReconfigurableDispatcher = flagValue(source, appId, version, Flags.USE_RECONFIGURABLE_DISPATCHER); this.contentLayerMetadataFeatureLevel = flagValue(source, appId, version, Flags.CONTENT_LAYER_METADATA_FEATURE_LEVEL); + this.dynamicHeapSize = flagValue(source, appId, version, Flags.DYNAMIC_HEAP_SIZE); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -293,6 +295,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean enableNestedMultivalueGrouping() { return enableNestedMultivalueGrouping; } @Override public boolean useReconfigurableDispatcher() { return useReconfigurableDispatcher; } @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } + @Override public boolean dynamicHeapSize() { return dynamicHeapSize; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java index 4c262379c35..1288b63cadd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java @@ -111,6 +111,12 @@ public class ZKApplication { return getBytesInternal(getFullPath(path)); } + public long getSize(Path path) { + return curator.getStat(path).map(stat -> (long)stat.getDataLength()) + .orElseThrow(() -> new IllegalArgumentException( + "Could not get size from '" + path + "' in zookeeper")); + } + void putData(Path path, String data) { byte[] bytes = Utf8.toBytes(data); ensureDataIsNotTooLarge(bytes, path); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java index 6bc29331efb..e51f8627de2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java @@ -3,8 +3,9 @@ package com.yahoo.vespa.config.server.zookeeper; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.path.Path; import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.util.ConfigUtils; import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; @@ -13,11 +14,9 @@ import java.io.InputStream; import java.io.Reader; import java.io.StringReader; import java.io.StringWriter; -import java.util.logging.Level; -import com.yahoo.vespa.config.util.ConfigUtils; - import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH; @@ -184,6 +183,8 @@ class ZKApplicationFile extends ApplicationFile { } } + @Override public long getSize() { return zkApp.getSize(getZKPath(path)); } + @Override public int compareTo(ApplicationFile other) { if (other == this) return 0; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index ca2f9da3273..8d1b0fefbaf 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -50,10 +50,11 @@ import static org.junit.Assert.assertTrue; public class LbServicesProducerTest { private static final Set<ContainerEndpoint> endpoints = Set.of( + new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("mydisc.foo.foo.endpoint1.suffix")), + new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("mydisc.foo.foo.endpoint2.suffix")), new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.global, List.of("rotation-1", "rotation-2")), new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.application, List.of("app-endpoint")) ); - private static final List<String> zoneDnsSuffixes = List.of(".endpoint1.suffix", ".endpoint2.suffix"); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); @@ -228,7 +229,7 @@ public class LbServicesProducerTest { private TestProperties getTestproperties(ApplicationId applicationId) { return new TestProperties() .setHostedVespa(true) - .setZoneDnsSuffixes(zoneDnsSuffixes) .setApplicationId(applicationId); } + } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java index 3edad64f9f2..3dfa278662d 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java @@ -135,14 +135,14 @@ public class NGramSearcher extends Searcher { * Creates the root of the query subtree which will contain the grams to match, * called by {@link #splitToGrams}. This hook is provided to make it easy to create a subclass which * matches grams using a different composite item, e.g an OrItem. - * <p> + * * This default implementation returns createGramRoot(query). * * @param term the term item this gram root is replacing in the query tree, * typically used to access the index name of the term when that is required by the new gram root * (such as in PhraseItem) * @param query the input query, to make it possible to return a different composite item type - * depending on the query content + * depending on the query content * @return the composite item to add the gram items to in {@link #splitToGrams} */ protected CompositeItem createGramRoot(HasIndexItem term, Query query) { diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/EquivTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/EquivTestCase.java new file mode 100644 index 00000000000..3e2c634b7f2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/EquivTestCase.java @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import org.junit.jupiter.api.Test; + +/** + * @author bratseth + */ +public class EquivTestCase extends RuleBaseAbstractTestCase { + + public EquivTestCase() { + super("equiv.sr"); + } + + @Test + void testEquiv() { + assertSemantics("EQUIV \"lord of the rings\" lotr", "lotr"); + } + + @Test + void testEquivWithFollowingQuery() { + assertSemantics("AND (EQUIV \"lord of the rings\" lotr) is a movie", "lotr is a movie"); + } + + @Test + void testEquivWithPrecedingQuery() { + assertSemantics("AND a movie is (EQUIV \"lord of the rings\" lotr)", "a movie is lotr"); + } + + @Test + void testEquivWithSurroundingQuery() { + assertSemantics("AND a movie is (EQUIV \"lord of the rings\" lotr) yes", "a movie is lotr yes"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java index 49449153d1f..4d25ebdccff 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java @@ -334,15 +334,15 @@ public class NGramSearcherTestCase { Result r = new Execution(new Chain<>(createSearcher(), new MockBackend1()), createContextStub(createIndexFacts())).search(q); Hit h1 = r.hits().get("hit1"); assertEquals("Should be untouched,\u001feven if containing \u001f", - h1.getField("test").toString()); + h1.getField("test").toString()); assertTrue(h1.getField("test") instanceof String); assertEquals("Blue red Ed A", h1.getField("gram2").toString()); assertTrue(h1.getField("gram2") instanceof XMLString); assertEquals("Blue red ed a\u001f", - h1.getField("gram3").toString(), - "Separators on borders work"); + h1.getField("gram3").toString(), + "Separators on borders work"); assertTrue(h1.getField("gram3") instanceof String); Hit h2 = r.hits().get("hit2"); @@ -352,7 +352,7 @@ public class NGramSearcherTestCase { Hit h3 = r.hits().get("hit3"); assertEquals("\u001ffin\u001f \u001fen\u001f \u001fa\u001f", h3.getField("gram2").toString()); assertEquals("#Logging in #Java is like that \"Judean P\u001fopul\u001far Front\" scene from \"Life of Brian\".", - h3.getField("gram3").toString()); + h3.getField("gram3").toString()); } private Item parseQuery(String query, Query.Type type) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index e661c88e117..856af9f4132 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -258,15 +258,15 @@ public class SystemFlagsDataArchive { root = mapper.readTree(fileContent); // TODO (mortent): Remove this after completing migration of APPLICATION_ID dimension // replace "application" with "instance" for all dimension fields -// List<JsonNode> dimensionParents = root.findParents("dimension"); -// for (JsonNode parentNode : dimensionParents) { -// JsonNode dimension = parentNode.get("dimension"); -// if (dimension.isTextual() && "application".equals(dimension.textValue())) { -// ObjectNode parent = (ObjectNode) parentNode; -// parent.remove("dimension"); -// parent.put("dimension", "instance"); -// } -// } + List<JsonNode> dimensionParents = root.findParents("dimension"); + for (JsonNode parentNode : dimensionParents) { + JsonNode dimension = parentNode.get("dimension"); + if (dimension.isTextual() && "application".equals(dimension.textValue())) { + ObjectNode parent = (ObjectNode) parentNode; + parent.remove("dimension"); + parent.put("dimension", "instance"); + } + } } catch (JsonProcessingException e) { throw new FlagValidationException("Invalid JSON: " + e.getMessage()); } diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index aba6cfbfeac..373f8ba9de2 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -245,7 +245,7 @@ public class SystemFlagsDataArchiveTest { "conditions": [ { "type": "whitelist", - "dimension": "application", + "dimension": "instance", "values": [ "f:o:o" ] } ], @@ -287,7 +287,7 @@ public class SystemFlagsDataArchiveTest { { "comment": "bar", "type": "whitelist", - "dimension": "application", + "dimension": "instance", "values": [ "f:o:o" ] } ], @@ -308,7 +308,7 @@ public class SystemFlagsDataArchiveTest { @Test void normalize_json_succeed_on_valid_values() { addFile(Condition.Type.WHITELIST, "application", "a:b:c"); -// addFile(Condition.Type.WHITELIST, "instance", "a:b:c"); + addFile(Condition.Type.WHITELIST, "instance", "a:b:c"); addFile(Condition.Type.WHITELIST, "cloud", "yahoo"); addFile(Condition.Type.WHITELIST, "cloud", "aws"); addFile(Condition.Type.WHITELIST, "cloud", "gcp"); @@ -362,7 +362,7 @@ public class SystemFlagsDataArchiveTest { @Test void normalize_json_fail_on_invalid_values() { - failAddFile(Condition.Type.WHITELIST, "application", "a.b.c", "In file flags/temporary/foo/default.json: Invalid application 'a.b.c' in whitelist condition: Application ids must be on the form tenant:application:instance, but was a.b.c"); + failAddFile(Condition.Type.WHITELIST, "application", "a.b.c", "In file flags/temporary/foo/default.json: Invalid instance 'a.b.c' in whitelist condition: Application ids must be on the form tenant:application:instance, but was a.b.c"); failAddFile(Condition.Type.WHITELIST, "cloud", "foo", "In file flags/temporary/foo/default.json: Unknown cloud: foo"); // cluster-id: any String is valid failAddFile(Condition.Type.WHITELIST, "cluster-type", "foo", "In file flags/temporary/foo/default.json: Invalid cluster-type 'foo' in whitelist condition: Illegal cluster type 'foo'"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 091836a1eea..27cd5e7e576 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -525,7 +525,8 @@ public class RoutingController { } public boolean generatedEndpointsEnabled(ApplicationId instance) { - return randomizedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()).value(); + return randomizedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) + .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()).value(); } private static void requireGeneratedEndpoints(GeneratedEndpointList generatedEndpoints, boolean declared) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index e01da00a27e..d661fa189b9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -140,10 +140,11 @@ public class EndpointCertificates { } try (NestedTransaction transaction = new NestedTransaction()) { curator.removeUnassignedCertificate(candidate.get(), transaction); - curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, candidate.get().certificate()), + EndpointCertificate certificate = candidate.get().certificate().withLastRequested(clock.instant().getEpochSecond()); + curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, certificate), transaction); transaction.commit(); - return candidate.get().certificate(); + return certificate; } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java index 70eeb2b9f6c..ed383175cc3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java @@ -69,7 +69,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { // Create metric for available certificates in the pool as a fraction of configured size int poolSize = certPoolSize.value(); long available = certificatePool.stream().filter(c -> c.state() == UnassignedCertificate.State.ready).count(); - metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? (available/poolSize) : 1.0), metric.createContext(Map.of())); + metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? ((double)available/poolSize) : 1.0), metric.createContext(Map.of())); if (certificatePool.size() < poolSize) { provisionRandomizedCertificate(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index 1cb43453918..a6d3b435dcb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -306,6 +306,9 @@ public class EndpointCertificatesTest { fail("Expected exception as certificate is not ready"); } catch (IllegalArgumentException ignored) {} + // Advance clock to verify last requested time + clock.advance(Duration.ofDays(3)); + // Certificate is assigned from pool instead. The previously assigned certificate will eventually be cleaned up // by EndpointCertificateMaintainer { // prod @@ -315,6 +318,7 @@ public class EndpointCertificatesTest { assertEquals(certId, cert.get().randomizedId().get()); assertEquals(certId, tester.curator().readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.empty()).get().certificate().randomizedId().get(), "Certificate is assigned at application-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); + assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); } { // dev @@ -325,6 +329,7 @@ public class EndpointCertificatesTest { assertEquals(certId, cert.get().randomizedId().get()); assertEquals(certId, tester.curator().readAssignedCertificate(instance.id()).get().certificate().randomizedId().get(), "Certificate is assigned at instance-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); + assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java index 88c5ae9ff06..4257261b09b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java @@ -53,12 +53,4 @@ public class CertificatePoolMaintainerTest { assertEquals(0.0, maintainer.maintain(), 0.0000001); assertEquals(n, tester.curator().readUnassignedCertificates().size()); } - - void old_unassigned_certs_are_refreshed() { - tester.flagSource().withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), 1); - assertNumCerts(1); - EndpointCertificateProviderMock endpointCertificateProvider = (EndpointCertificateProviderMock) tester.controller().serviceRegistry().endpointCertificateProvider(); - var request = endpointCertificateProvider.listCertificates().get(0); - - } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java index 779aee73dae..eb3f9daef53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java @@ -63,7 +63,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\"}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email // Resolved for email, but conditions are empty since this user is not authorized for any tenants - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\"}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\"}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email flagData, Set.of(), false, email1); @@ -72,7 +72,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\",\"values\":[\"tenant1\"]}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email // Resolved for email, but conditions have filtered out tenant2 - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email flagData, Set.of("tenant1"), false, email1); @@ -81,7 +81,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Default from code, no DB values match // Includes last value from DB which is not conditioned on email and the default from code - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\",\"tenant2:music:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\",\"tenant2:music:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"default value\"}]}]}", // Default from code flagData, Set.of(), true, "operator@domain.tld"); } diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 962d666bf6b..120c6104793 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -34,7 +34,7 @@ <!-- DO NOT UPGRADE THESE TO A NEW MAJOR VERSION WITHOUT CHECKING FOR BINARY COMPATIBILITY --> <aopalliance.vespa.version>1.0</aopalliance.vespa.version> <commons-logging.vespa.version>1.2</commons-logging.vespa.version> <!-- This version is exported by jdisc via jcl-over-slf4j. --> - <error-prone-annotations.vespa.version>2.21.1</error-prone-annotations.vespa.version> + <error-prone-annotations.vespa.version>2.22.0</error-prone-annotations.vespa.version> <guava.vespa.version>32.1.2-jre</guava.vespa.version> <guice.vespa.version>6.0.0</guice.vespa.version> <jackson2.vespa.version>2.15.2</jackson2.vespa.version> @@ -123,7 +123,7 @@ <protobuf.vespa.version>3.24.3</protobuf.vespa.version> <questdb.vespa.version>7.3.2</questdb.vespa.version> <spifly.vespa.version>1.3.6</spifly.vespa.version> - <snappy.vespa.version>1.1.10.3</snappy.vespa.version> + <snappy.vespa.version>1.1.10.4</snappy.vespa.version> <surefire.vespa.version>3.1.2</surefire.vespa.version> <wiremock.vespa.version>3.1.0</wiremock.vespa.version> <xerces.vespa.version>2.12.2</xerces.vespa.version> @@ -153,7 +153,7 @@ <maven-plugin-api.vespa.version>${maven-core.vespa.version}</maven-plugin-api.vespa.version> <maven-plugin-tools.vespa.version>3.9.0</maven-plugin-tools.vespa.version> <maven-resources-plugin.vespa.version>3.3.1</maven-resources-plugin.vespa.version> - <maven-shade-plugin.vespa.version>3.5.0</maven-shade-plugin.vespa.version> + <maven-shade-plugin.vespa.version>3.5.1</maven-shade-plugin.vespa.version> <maven-site-plugin.vespa.version>3.12.1</maven-site-plugin.vespa.version> <maven-source-plugin.vespa.version>3.3.0</maven-source-plugin.vespa.version> <properties-maven-plugin.vespa.version>1.2.0</properties-maven-plugin.vespa.version> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 2e158f0f3ef..e5b76bedecd 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -406,6 +406,13 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); + public static final UnboundBooleanFlag DYNAMIC_HEAP_SIZE = defineFeatureFlag( + "dynamic-heap-size", false, + List.of("bjorncs"), "2023-09-21", "2024-01-15", + "Whether to calculate JVM heap size based on predicted Onnx model memory requirements", + "Takes effect at redeployment", + INSTANCE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java index f867daac245..8fb48c8a82f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java @@ -23,7 +23,7 @@ public class DimensionHelper { serializedDimensions.put(FetchVector.Dimension.CONSOLE_USER_EMAIL, List.of("console-user-email")); serializedDimensions.put(FetchVector.Dimension.ENVIRONMENT, List.of("environment")); serializedDimensions.put(FetchVector.Dimension.HOSTNAME, List.of("hostname")); - serializedDimensions.put(FetchVector.Dimension.INSTANCE_ID, List.of("application", "instance")); + serializedDimensions.put(FetchVector.Dimension.INSTANCE_ID, List.of("instance", "application")); serializedDimensions.put(FetchVector.Dimension.NODE_TYPE, List.of("node-type")); serializedDimensions.put(FetchVector.Dimension.SYSTEM, List.of("system")); serializedDimensions.put(FetchVector.Dimension.TENANT_ID, List.of("tenant")); diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 970bb6732a1..6853632ea40 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -7,7 +7,7 @@ com.fasterxml.jackson.core:jackson-annotations:2.15.2 com.fasterxml.jackson.core:jackson-core:2.15.2 com.fasterxml.jackson.core:jackson-databind:2.15.2 com.github.luben:zstd-jni:1.5.5-5 -com.google.errorprone:error_prone_annotations:2.21.1 +com.google.errorprone:error_prone_annotations:2.22.0 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:32.1.2-jre com.google.inject:guice:6.0.0 @@ -35,7 +35,7 @@ org.apache.maven:maven-settings-builder:3.9.4 org.apache.maven.enforcer:enforcer-api:3.4.1 org.apache.maven.enforcer:enforcer-rules:3.4.1 org.apache.maven.plugin-tools:maven-plugin-annotations:3.9.0 -org.apache.maven.plugins:maven-shade-plugin:3.5.0 +org.apache.maven.plugins:maven-shade-plugin:3.5.1 org.apache.maven.resolver:maven-resolver-api:1.9.14 org.apache.maven.resolver:maven-resolver-impl:1.9.14 org.apache.maven.resolver:maven-resolver-named-locks:1.9.14 @@ -63,7 +63,7 @@ org.ow2.asm:asm-tree:9.5 org.slf4j:slf4j-api:1.7.36 org.tukaani:xz:1.9 org.twdata.maven:mojo-executor:2.4.0 -org.vafer:jdependency:2.8.0 +org.vafer:jdependency:2.9.0 #[test-only] # Contains dependencies that are used exclusively in 'test' scope diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index 0300d7e92ff..d902fb7b3c4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; import java.net.URI; @@ -73,9 +74,7 @@ public class NodeSpec { private final List<TrustStoreItem> trustStore; - private final Optional<WireguardKey> wireguardPubkey; - - private final Optional<Instant> wireguardKeyTimestamp; + private final Optional<WireguardKeyWithTimestamp> wireguardKeyWithTimestamp; private final boolean wantToRebuild; @@ -112,8 +111,7 @@ public class NodeSpec { Optional<URI> archiveUri, Optional<ApplicationId> exclusiveTo, List<TrustStoreItem> trustStore, - Optional<WireguardKey> wireguardPubkey, - Optional<Instant> wireguardKeyTimestamp, + Optional<WireguardKeyWithTimestamp> wireguardPubkey, boolean wantToRebuild) { if (state == NodeState.active) { @@ -157,8 +155,7 @@ public class NodeSpec { this.archiveUri = Objects.requireNonNull(archiveUri); this.exclusiveTo = Objects.requireNonNull(exclusiveTo); this.trustStore = Objects.requireNonNull(trustStore); - this.wireguardPubkey = Objects.requireNonNull(wireguardPubkey); - this.wireguardKeyTimestamp = Objects.requireNonNull(wireguardKeyTimestamp); + this.wireguardKeyWithTimestamp = Objects.requireNonNull(wireguardPubkey); this.wantToRebuild = wantToRebuild; } @@ -313,9 +310,7 @@ public class NodeSpec { return trustStore; } - public Optional<WireguardKey> wireguardPubkey() { return wireguardPubkey; } - - public Optional<Instant> wireguardKeyTimestamp() { return wireguardKeyTimestamp; } + public Optional<WireguardKeyWithTimestamp> wireguardKeyWithTimestamp() { return wireguardKeyWithTimestamp; } public boolean wantToRebuild() { return wantToRebuild; @@ -358,8 +353,7 @@ public class NodeSpec { Objects.equals(archiveUri, that.archiveUri) && Objects.equals(exclusiveTo, that.exclusiveTo) && Objects.equals(trustStore, that.trustStore) && - Objects.equals(wireguardPubkey, that.wireguardPubkey) && - Objects.equals(wireguardKeyTimestamp, that.wireguardKeyTimestamp) && + Objects.equals(wireguardKeyWithTimestamp, that.wireguardKeyWithTimestamp) && Objects.equals(wantToRebuild, that.wantToRebuild); } @@ -398,8 +392,7 @@ public class NodeSpec { archiveUri, exclusiveTo, trustStore, - wireguardPubkey, - wireguardKeyTimestamp, + wireguardKeyWithTimestamp, wantToRebuild); } @@ -438,8 +431,7 @@ public class NodeSpec { + " archiveUri=" + archiveUri + " exclusiveTo=" + exclusiveTo + " trustStore=" + trustStore - + " wireguardPubkey=" + wireguardPubkey - + " wireguardKeyTimestamp=" + wireguardKeyTimestamp + + " wireguardPubkey=" + wireguardKeyWithTimestamp + " wantToRebuild=" + wantToRebuild + " }"; } @@ -477,8 +469,7 @@ public class NodeSpec { private Optional<URI> archiveUri = Optional.empty(); private Optional<ApplicationId> exclusiveTo = Optional.empty(); private List<TrustStoreItem> trustStore = List.of(); - private Optional<WireguardKey> wireguardPubkey = Optional.empty(); - private Optional<Instant> wireguardKeyTimestamp = Optional.empty(); + private Optional<WireguardKeyWithTimestamp> wireguardPubkey = Optional.empty(); private boolean wantToRebuild = false; public Builder() {} @@ -514,8 +505,7 @@ public class NodeSpec { node.archiveUri.ifPresent(this::archiveUri); node.exclusiveTo.ifPresent(this::exclusiveTo); trustStore(node.trustStore); - node.wireguardPubkey.ifPresent(this::wireguardPubkey); - node.wireguardKeyTimestamp.ifPresent(this::wireguardKeyTimestamp); + node.wireguardKeyWithTimestamp.ifPresent(this::wireguardKeyWithTimestamp); wantToRebuild(node.wantToRebuild); } @@ -704,13 +694,13 @@ public class NodeSpec { return this; } - public Builder wireguardPubkey(WireguardKey wireguardPubKey) { - this.wireguardPubkey = Optional.of(wireguardPubKey); + public Builder wireguardPubkey(WireguardKey wireguardPubkey) { + this.wireguardPubkey = Optional.of(new WireguardKeyWithTimestamp(wireguardPubkey, Instant.EPOCH)); return this; } - public Builder wireguardKeyTimestamp(Instant wireguardKeyTimestamp) { - this.wireguardKeyTimestamp = Optional.of(wireguardKeyTimestamp); + public Builder wireguardKeyWithTimestamp(WireguardKeyWithTimestamp wireguardPubKey) { + this.wireguardPubkey = Optional.of(wireguardPubKey); return this; } @@ -846,7 +836,7 @@ public class NodeSpec { wantedFirmwareCheck, currentFirmwareCheck, modelName, resources, realResources, ipAddresses, additionalIpAddresses, reports, events, parentHostname, archiveUri, exclusiveTo, trustStore, - wireguardPubkey, wireguardKeyTimestamp, wantToRebuild); + wireguardPubkey, wantToRebuild); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index a9cc2d698e9..17d3b51398f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; @@ -139,26 +140,28 @@ public class RealNodeRepository implements NodeRepository { return response.nodes.stream() .mapMulti((NodeRepositoryNode node, Consumer<WireguardPeer> consumer) -> { - if (node.wireguardPubkey == null || node.wireguardPubkey.isEmpty()) return; - List<VersionedIpAddress> ipAddresses = node.ipAddresses.stream() - .map(InetAddresses::forString) - .filter(address -> !address.isLoopbackAddress() && !address.isLinkLocalAddress() && !address.isSiteLocalAddress()) - .map(VersionedIpAddress::from) - .toList(); - if (ipAddresses.isEmpty()) return; + var keyWithTimestamp = createWireguardKeyWithTimestamp(node.wireguardKeyWithTimestamp, + node.wireguardPubkey, + node.wireguardKeyTimestamp); + if (keyWithTimestamp == null) return; - // Unbox to prevent NPE - long keyTimestamp = node.wireguardKeyTimestamp == null ? 0L : node.wireguardKeyTimestamp; + List<VersionedIpAddress> ipAddresses = getIpAddresses(node); + if (ipAddresses.isEmpty()) return; - consumer.accept(new WireguardPeer(HostName.of(node.hostname), - ipAddresses, - WireguardKey.from(node.wireguardPubkey), - Instant.ofEpochMilli(keyTimestamp))); + consumer.accept(new WireguardPeer(HostName.of(node.hostname), ipAddresses, keyWithTimestamp)); }) .sorted() .toList(); } + private static List<VersionedIpAddress> getIpAddresses(NodeRepositoryNode node) { + return node.ipAddresses.stream() + .map(InetAddresses::forString) + .filter(address -> !address.isLoopbackAddress() && !address.isLinkLocalAddress() && !address.isSiteLocalAddress()) + .map(VersionedIpAddress::from) + .toList(); + } + @Override public List<WireguardPeer> getConfigserverPeers() { GetWireguardResponse response = configServerApi.get("/nodes/v2/wireguard", GetWireguardResponse.class); @@ -246,8 +249,9 @@ public class RealNodeRepository implements NodeRepository { Optional.ofNullable(node.archiveUri).map(URI::create), Optional.ofNullable(node.exclusiveTo).map(ApplicationId::fromSerializedForm), trustStore, - Optional.ofNullable(node.wireguardPubkey).map(WireguardKey::from), - Optional.ofNullable(node.wireguardKeyTimestamp).map(Instant::ofEpochMilli), + Optional.ofNullable(createWireguardKeyWithTimestamp(node.wireguardKeyWithTimestamp, + node.wireguardPubkey, + node.wireguardKeyTimestamp)), node.wantToRebuild); } @@ -364,20 +368,39 @@ public class RealNodeRepository implements NodeRepository { node.trustStore = nodeAttributes.getTrustStore().stream() .map(item -> new NodeRepositoryNode.TrustStoreItem(item.fingerprint(), item.expiry().toEpochMilli())) .toList(); - node.wireguardPubkey = nodeAttributes.getWireguardPubkey().map(WireguardKey::value).orElse(null); + // This is used for patching, and timestamp must only be set on the server side, hence sending EPOCH. + node.wireguardKeyWithTimestamp = nodeAttributes.getWireguardPubkey() + .map(key -> new NodeRepositoryNode.WireguardKeyWithTimestamp(key.value(), 0L)) + .orElse(null); Map<String, JsonNode> reports = nodeAttributes.getReports(); node.reports = reports == null || reports.isEmpty() ? null : new TreeMap<>(reports); + // TODO wg: remove when all nodes are using new key+timestamp format + node.wireguardPubkey = nodeAttributes.getWireguardPubkey().map(WireguardKey::value).orElse(null); return node; } private static WireguardPeer createConfigserverPeer(GetWireguardResponse.Configserver configServer) { - // Unbox to prevent NPE - long keyTimestamp = configServer.wireguardKeyTimestamp == null ? 0L : configServer.wireguardKeyTimestamp; - return new WireguardPeer(HostName.of(configServer.hostname), configServer.ipAddresses.stream().map(VersionedIpAddress::from).toList(), - WireguardKey.from(configServer.wireguardPubkey), - Instant.ofEpochMilli(keyTimestamp)); + createWireguardKeyWithTimestamp(configServer.wireguardKeyWithTimestamp, + configServer.wireguardPubkey, + configServer.wireguardKeyTimestamp)); + } + + private static WireguardKeyWithTimestamp createWireguardKeyWithTimestamp(NodeRepositoryNode.WireguardKeyWithTimestamp wirguardJson, + String oldKeyJson, Long oldTimestampJson) { + if (wirguardJson != null && wirguardJson.key != null && ! wirguardJson.key.isEmpty()) { + return new WireguardKeyWithTimestamp(WireguardKey.from(wirguardJson.key), + Instant.ofEpochMilli(wirguardJson.timestamp)); + // TODO wg: remove when all nodes are using new key+timestamp format + } else if (oldKeyJson != null) { + var timestamp = oldTimestampJson != null ? oldTimestampJson : 0L; + return new WireguardKeyWithTimestamp(WireguardKey.from(oldKeyJson), + Instant.ofEpochMilli(timestamp)); + // TODO END + } else return null; + } + } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java index dcbf4cc163f..47903795ef7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java @@ -27,27 +27,23 @@ public class GetWireguardResponse { public static class Configserver { @JsonProperty("hostname") - public final String hostname; + public String hostname; @JsonProperty("ipAddresses") - public final List<String> ipAddresses; + public List<String> ipAddresses; + + @JsonProperty("wireguard") + public NodeRepositoryNode.WireguardKeyWithTimestamp wireguardKeyWithTimestamp; - @JsonProperty("wireguardPubkey") - public final String wireguardPubkey; + // TODO wg: remove when all nodes use new key+timestamp format + @JsonProperty("wireguardPubkey") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public String wireguardPubkey; @JsonProperty("wireguardKeyTimestamp") - public final Long wireguardKeyTimestamp; - - @JsonCreator - public Configserver(@JsonProperty("hostname") String hostname, - @JsonProperty("ipAddresses") List<String> ipAddresses, - @JsonProperty("wireguardPubkey") String wireguardPubkey, - @JsonProperty("wireguardKeyTimestamp") Long wireguardKeyTimestamp) { - this.hostname = hostname; - this.ipAddresses = ipAddresses; - this.wireguardPubkey = wireguardPubkey; - this.wireguardKeyTimestamp = wireguardKeyTimestamp; - } + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public Long wireguardKeyTimestamp; + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java index 3d0d052a877..35ca757ebbe 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java @@ -92,6 +92,10 @@ public class NodeRepositoryNode { @JsonProperty("trustStore") @JsonInclude(JsonInclude.Include.NON_EMPTY) public List<TrustStoreItem> trustStore; + @JsonProperty("wireguard") + public WireguardKeyWithTimestamp wireguardKeyWithTimestamp; + + // TODO wg: remove separate key and timestamp when all nodes use new keyWithTimestamp @JsonProperty("wireguardPubkey") @JsonInclude(JsonInclude.Include.NON_EMPTY) public String wireguardPubkey; @@ -141,13 +145,25 @@ public class NodeRepositoryNode { ", exclusiveTo='" + exclusiveTo + '\'' + ", history=" + history + ", trustStore=" + trustStore + - ", wireguardPubkey=" + wireguardPubkey + - ", wireguardKeyTimestamp=" + wireguardKeyTimestamp + + ", wireguard=" + wireguardKeyTimestamp + ", reports=" + reports + '}'; } @JsonIgnoreProperties(ignoreUnknown = true) + public static class WireguardKeyWithTimestamp { + @JsonProperty("key") + public String key; + @JsonProperty("timestamp") + public long timestamp; + + public WireguardKeyWithTimestamp(@JsonProperty("key") String key, @JsonProperty("timestamp") long timestamp) { + this.key = key; + this.timestamp = timestamp; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) public static class Owner { @JsonProperty("tenant") public String tenant; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java index b5428f57f08..e5ab9a1ce31 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java @@ -1,10 +1,9 @@ package com.yahoo.vespa.hosted.node.admin.wireguard; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; -import java.time.Instant; import java.util.List; /** @@ -15,8 +14,7 @@ import java.util.List; */ public record WireguardPeer(HostName hostname, List<VersionedIpAddress> ipAddresses, - WireguardKey publicKey, - Instant wireguardKeyTimestamp) implements Comparable<WireguardPeer> { + WireguardKeyWithTimestamp keyWithTimestamp) implements Comparable<WireguardPeer> { public WireguardPeer { if (ipAddresses.isEmpty()) throw new IllegalArgumentException("No IP addresses for peer node " + hostname.value()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 98e65d03f2f..ee3eac22d02 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; @@ -140,6 +141,7 @@ public class RealNodeRepositoryTest { var dockerImage = "registry.example.com/repo/image-1:6.2.3"; var wireguardKey = WireguardKey.from("111122223333444455556666777788889999000042c="); var wireguardKeyTimestamp = Instant.ofEpochMilli(123L); // Instant from clock in MockNodeRepository + var keyWithTimestamp = new WireguardKeyWithTimestamp(wireguardKey, wireguardKeyTimestamp); nodeRepositoryApi.updateNodeAttributes( hostname, @@ -151,8 +153,7 @@ public class RealNodeRepositoryTest { NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode(hostname).orElseThrow(); assertEquals(1, hostSpec.currentRestartGeneration().orElseThrow()); assertEquals(dockerImage, hostSpec.currentDockerImage().orElseThrow().asString()); - assertEquals(wireguardKey.value(), hostSpec.wireguardPubkey().orElseThrow().value()); - assertEquals(wireguardKeyTimestamp, hostSpec.wireguardKeyTimestamp().orElseThrow()); + assertEquals(keyWithTimestamp, hostSpec.wireguardKeyWithTimestamp().orElseThrow()); } @Test @@ -215,7 +216,7 @@ public class RealNodeRepositoryTest { assertWireguardPeer(cfgPeers.get(0), "cfg1.yahoo.com", "::201:1", "lololololololololololololololololololololoo=", - Instant.ofEpochMilli(456L)); + 456L); //// Exclave nodes //// @@ -227,16 +228,17 @@ public class RealNodeRepositoryTest { assertWireguardPeer(exclavePeers.get(0), "dockerhost2.yahoo.com", "::101:1", "000011112222333344445555666677778888999900c=", - Instant.ofEpochMilli(123L)); + 123L); } private void assertWireguardPeer(WireguardPeer peer, String hostname, String ipv6, - String publicKey, Instant keyTimestamp) { + String publicKey, long keyTimestamp) { assertEquals(hostname, peer.hostname().value()); assertEquals(1, peer.ipAddresses().size()); assertIp(peer.ipAddresses().get(0), ipv6, 6); - assertEquals(publicKey, peer.publicKey().value()); - assertEquals(keyTimestamp, peer.wireguardKeyTimestamp()); + var expectedKeyWithTimestamp = new WireguardKeyWithTimestamp(WireguardKey.from(publicKey), + Instant.ofEpochMilli(keyTimestamp)); + assertEquals(expectedKeyWithTimestamp, peer.keyWithTimestamp()); } private void assertIp(VersionedIpAddress ip, String expectedIp, int expectedVersion) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java index cd76b221c9e..6ee896e3db6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.wireguard; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; import org.junit.jupiter.api.Test; @@ -31,6 +32,7 @@ public class WireguardPeerTest { private static WireguardPeer peer(String hostname) { return new WireguardPeer(HostName.of(hostname), List.of(VersionedIpAddress.from("::1:1")), - WireguardKey.generateRandomForTesting(), Instant.EPOCH); + new WireguardKeyWithTimestamp(WireguardKey.generateRandomForTesting(), Instant.EPOCH)); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 24159b88a9b..d5e891a33c7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancers; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -64,8 +64,7 @@ public final class Node implements Nodelike { private final CloudAccount cloudAccount; /** Only set for configservers and exclave nodes */ - private final Optional<WireguardKey> wireguardPubKey; - private final Optional<Instant> wireguardKeyTimestamp; + private final Optional<WireguardKeyWithTimestamp> wireguardPubKey; /** Record of the last event of each type happening to this node */ private final History history; @@ -96,8 +95,8 @@ public final class Node implements Nodelike { NodeType type, Reports reports, Optional<String> modelName, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveToApplicationId, Optional<Duration> hostTTL, Optional<Instant> hostEmptyAt, Optional<ClusterSpec.Type> exclusiveToClusterType, Optional<String> switchHostname, - List<TrustStoreItem> trustStoreItems, CloudAccount cloudAccount, Optional<WireguardKey> wireguardPubKey, - Optional<Instant> wireguardKeyTimestamp) { + List<TrustStoreItem> trustStoreItems, CloudAccount cloudAccount, + Optional<WireguardKeyWithTimestamp> wireguardPubKey) { this.id = Objects.requireNonNull(id, "A node must have an ID"); this.extraId = Objects.requireNonNull(extraId, "Extra ID cannot be null"); this.hostname = requireNonEmptyString(hostname, "A node must have a hostname"); @@ -120,7 +119,6 @@ public final class Node implements Nodelike { this.trustStoreItems = Objects.requireNonNull(trustStoreItems).stream().distinct().toList(); this.cloudAccount = Objects.requireNonNull(cloudAccount); this.wireguardPubKey = Objects.requireNonNull(wireguardPubKey); - this.wireguardKeyTimestamp = Objects.requireNonNull(wireguardKeyTimestamp); if (state == State.active) requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); @@ -264,15 +262,10 @@ public final class Node implements Nodelike { } /** Returns the wireguard public key of this node. Only relevant for enclave nodes. */ - public Optional<WireguardKey> wireguardPubKey() { + public Optional<WireguardKeyWithTimestamp> wireguardPubKey() { return wireguardPubKey; } - /** Returns the timestamp of the wireguard key of this node. Only relevant for enclave nodes. */ - public Optional<Instant> wireguardKeyTimestamp() { - return wireguardKeyTimestamp; - } - /** * Returns a copy of this where wantToFail is set to true and history is updated to reflect this. */ @@ -367,16 +360,14 @@ public final class Node implements Nodelike { public Node with(Status status) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a node with the flavor assigned to the given value */ @@ -385,40 +376,35 @@ public final class Node implements Nodelike { History updateHistory = history.with(new History.Event(History.Event.Type.resized, agent, instant)); return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, updateHistory, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with the reboot generation set to generation */ public Node withReboot(Generation generation) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with given id set */ public Node withId(String id) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with model name set to given value */ public Node withModelName(String modelName) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.of(modelName), reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with model name cleared */ public Node withoutModelName() { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.empty(), reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with a history record saying it was detected to be down at this instant */ @@ -460,24 +446,21 @@ public final class Node implements Nodelike { public Node with(Allocation allocation) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, Optional.of(allocation), history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node with IP config set to the given value. */ public Node with(IP.Config ipConfig) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { return new Node(id, extraId, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withReservedTo(TenantName tenant) { @@ -485,73 +468,59 @@ public final class Node implements Nodelike { throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type); return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, Optional.of(tenant), exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node which is not reserved to a tenant */ public Node withoutReservedTo() { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, Optional.empty(), exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExclusiveToApplicationId(ApplicationId exclusiveTo) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExtraId(Optional<String> extraId) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withHostTTL(Duration hostTTL) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, Optional.ofNullable(hostTTL), hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withHostEmptyAt(Instant hostEmptyAt) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, Optional.ofNullable(hostEmptyAt), - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExclusiveToClusterType(ClusterSpec.Type exclusiveTo) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - Optional.ofNullable(exclusiveTo), switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + Optional.ofNullable(exclusiveTo), switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } - public Node withWireguardPubkey(WireguardKey wireguardPubkey) { + public Node withWireguardPubkey(WireguardKeyWithTimestamp wireguardPubkey) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, Optional.ofNullable(wireguardPubkey), - wireguardKeyTimestamp); - } - - public Node withWireguardKeyTimestamp(Instant wireguardKeyTimestamp) { - return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - Optional.ofNullable(wireguardKeyTimestamp)); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, + Optional.ofNullable(wireguardPubkey)); } /** Returns a copy of this node with switch hostname set to given value */ public Node withSwitchHostname(String switchHostname) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, Optional.ofNullable(switchHostname), trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, Optional.ofNullable(switchHostname), trustStoreItems, cloudAccount, + wireguardPubKey); } /** Returns a copy of this node with switch hostname unset */ @@ -604,22 +573,19 @@ public final class Node implements Nodelike { public Node with(History history) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node with(Reports reports) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node with(List<TrustStoreItem> trustStoreItems) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } private static Optional<String> requireNonEmptyString(Optional<String> value, String message) { @@ -767,8 +733,7 @@ public final class Node implements Nodelike { private History history; private List<TrustStoreItem> trustStoreItems; private CloudAccount cloudAccount = CloudAccount.empty; - private WireguardKey wireguardPubKey; - private Instant wireguardKeyTimestamp; + private WireguardKeyWithTimestamp wireguardPubKey; private Builder(String id, String hostname, Flavor flavor, State state, NodeType type) { this.id = id; @@ -858,16 +823,11 @@ public final class Node implements Nodelike { return this; } - public Builder wireguardPubKey(WireguardKey wireguardPubKey) { + public Builder wireguardKey(WireguardKeyWithTimestamp wireguardPubKey) { this.wireguardPubKey = wireguardPubKey; return this; } - public Builder wireguardKeyTimestamp(Instant wireguardKeyTimestamp) { - this.wireguardKeyTimestamp = wireguardKeyTimestamp; - return this; - } - public Node build() { return new Node(id, Optional.empty(), Optional.ofNullable(ipConfig).orElse(IP.Config.EMPTY), hostname, Optional.ofNullable(parentHostname), flavor, Optional.ofNullable(status).orElseGet(Status::initial), state, Optional.ofNullable(allocation), @@ -875,7 +835,7 @@ public final class Node implements Nodelike { Optional.ofNullable(modelName), Optional.ofNullable(reservedTo), Optional.ofNullable(exclusiveToApplicationId), Optional.ofNullable(hostTTL), Optional.ofNullable(hostEmptyAt), Optional.ofNullable(exclusiveToClusterType), Optional.ofNullable(switchHostname), Optional.ofNullable(trustStoreItems).orElseGet(List::of), cloudAccount, - Optional.ofNullable(wireguardPubKey), Optional.ofNullable(wireguardKeyTimestamp)); + Optional.ofNullable(wireguardPubKey)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index 8ad975f5334..e4e08e5a15c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -223,7 +223,7 @@ public class CuratorDb { node.type(), node.reports(), node.modelName(), node.reservedTo(), node.exclusiveToApplicationId(), node.hostTTL(), node.hostEmptyAt(), node.exclusiveToClusterType(), node.switchHostname(), node.trustedCertificates(), - node.cloudAccount(), node.wireguardPubKey(), node.wireguardKeyTimestamp()); + node.cloudAccount(), node.wireguardPubKey()); curatorTransaction.add(createOrSet(nodePath(newNode), nodeSerializer.toJson(newNode))); writtenNodes.add(newNode); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 870e678a250..73531d650d5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.slime.ArrayTraverser; @@ -188,8 +189,10 @@ public class NodeSerializer { if (!node.cloudAccount().isUnspecified()) { object.setString(cloudAccountKey, node.cloudAccount().value()); } - node.wireguardPubKey().ifPresent(pubKey -> object.setString(wireguardPubKeyKey, pubKey.value())); - node.wireguardKeyTimestamp().ifPresent(timestamp -> object.setLong(wireguardKeyTimestampKey, timestamp.toEpochMilli())); + node.wireguardPubKey().ifPresent(pubKey -> { + object.setString(wireguardPubKeyKey, pubKey.key().value()); + object.setLong(wireguardKeyTimestampKey, pubKey.timestamp().toEpochMilli()); + }); } private void toSlime(Flavor flavor, Cursor object) { @@ -284,8 +287,7 @@ public class NodeSerializer { SlimeUtils.optionalString(object.field(switchHostnameKey)), trustedCertificatesFromSlime(object), SlimeUtils.optionalString(object.field(cloudAccountKey)).map(CloudAccount::from).orElse(CloudAccount.empty), - SlimeUtils.optionalString(object.field(wireguardPubKeyKey)).map(WireguardKey::from), - SlimeUtils.optionalInstant(object.field(wireguardKeyTimestampKey))); + wireguardKeyWithTimestampFromSlime(object.field(wireguardPubKeyKey), object.field(wireguardKeyTimestampKey))); } private Status statusFromSlime(Inspector object) { @@ -397,6 +399,13 @@ public class NodeSerializer { .toList(); } + private Optional<WireguardKeyWithTimestamp> wireguardKeyWithTimestampFromSlime(Inspector keyObject, Inspector timestampObject) { + if ( ! keyObject.valid()) return Optional.empty(); + return SlimeUtils.optionalString(keyObject).map( + key -> new WireguardKeyWithTimestamp(WireguardKey.from(key), + SlimeUtils.optionalInstant(timestampObject).orElse(null))); + } + // ----------------- Enum <-> string mappings ---------------------------------------- /** Returns the event type, or null if this event type should be ignored */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 9f1ab3dc3d5..cad034e01aa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; @@ -108,7 +109,8 @@ public class NodePatcher { "reports", "trustStore", "vespaVersion", - "wireguardPubkey")); + "wireguardPubkey", // TODO wg: remove when all nodes use new key+timestamp format + "wireguard")); if (!disallowedFields.isEmpty()) { throw new IllegalArgumentException("Patching fields not supported: " + disallowedFields); } @@ -271,9 +273,13 @@ public class NodePatcher { return value.type() == Type.NIX ? node.withoutSwitchHostname() : node.withSwitchHostname(value.asString()); case "trustStore": return nodeWithTrustStore(node, value); - case "wireguardPubkey": - return node.withWireguardPubkey(SlimeUtils.optionalString(value).map(WireguardKey::new).orElse(null)) - .withWireguardKeyTimestamp(clock.instant()); + case "wireguard": + // This is where we set the key timestamp. + var key = SlimeUtils.optionalString(value.field("key")).map(WireguardKey::new).orElse(null); + return node.withWireguardPubkey(new WireguardKeyWithTimestamp(key, clock.instant())); + case "wireguardPubkey": // TODO wg: remove when all nodes use new key+timestamp format + var oldKey = SlimeUtils.optionalString(value).map(WireguardKey::new).orElse(null); + return node.withWireguardPubkey(new WireguardKeyWithTimestamp(oldKey, clock.instant())); default: throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index a8f526544d7..05bb0a27d69 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.restapi.SlimeJsonResponse; @@ -192,8 +193,13 @@ class NodesResponse extends SlimeJsonResponse { if (!node.cloudAccount().isUnspecified()) { object.setString("cloudAccount", node.cloudAccount().value()); } - node.wireguardPubKey().ifPresent(key -> object.setString("wireguardPubkey", key.value())); - node.wireguardKeyTimestamp().ifPresent(timestamp -> object.setLong("wireguardKeyTimestamp", timestamp.toEpochMilli())); + node.wireguardPubKey().ifPresent(key -> toSlime(key, object.setObject("wireguard"))); + + // TODO wg: remove when all nodes have upgraded to new key+timestamp format + node.wireguardPubKey().ifPresent(key -> { + object.setString("wireguardPubkey", key.key().value()); + object.setLong("wireguardKeyTimestamp", key.timestamp().toEpochMilli()); + }); } private Version resolveVersionFlag(StringFlag flag, Node node, Allocation allocation) { @@ -237,6 +243,11 @@ class NodesResponse extends SlimeJsonResponse { } } + static void toSlime(WireguardKeyWithTimestamp keyWithTimestamp, Cursor object) { + object.setString("key", keyWithTimestamp.key().value()); + object.setLong("timestamp", keyWithTimestamp.timestamp().toEpochMilli()); + } + private Optional<DockerImage> currentContainerImage(Node node) { if (node.status().containerImage().isPresent()) { return node.status().containerImage(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java index 16e85dfa48a..e29c4f1b87a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java @@ -1,7 +1,7 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.hosted.provision.Node; @@ -10,9 +10,9 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.IP; import java.net.InetAddress; -import java.time.Instant; import java.util.List; -import java.util.Optional; + +import static com.yahoo.vespa.hosted.provision.restapi.NodesResponse.toSlime; /** * A response containing the wireguard peer config for each configserver that has a public key. @@ -36,17 +36,20 @@ public class WireguardResponse extends SlimeJsonResponse { .toList(); if (ipAddresses.isEmpty()) continue; - addConfigserver(cfgArray.addObject(), cfg.hostname(), cfg.wireguardPubKey().get(), - cfg.wireguardKeyTimestamp(), ipAddresses); + addConfigserver(cfgArray.addObject(), cfg.hostname(), cfg.wireguardPubKey().get(), ipAddresses); } } - private void addConfigserver(Cursor cfgEntry, String hostname, WireguardKey key, Optional<Instant> keyTimestamp, + private void addConfigserver(Cursor cfgEntry, String hostname, WireguardKeyWithTimestamp keyWithTimestamp, List<String> ipAddresses) { cfgEntry.setString("hostname", hostname); - cfgEntry.setString("wireguardPubkey", key.value()); - cfgEntry.setLong("wireguardKeyTimestamp", keyTimestamp.orElse(Instant.EPOCH).toEpochMilli()); + + // TODO wg: remove when all nodes are using new key+timestamp format + cfgEntry.setString("wireguardPubkey", keyWithTimestamp.key().value()); + cfgEntry.setLong("wireguardKeyTimestamp", keyWithTimestamp.timestamp().toEpochMilli()); + NodesResponse.ipAddressesToSlime(ipAddresses, cfgEntry.setArray("ipAddresses")); + toSlime(keyWithTimestamp, cfgEntry.setObject("wireguard")); } private static boolean isPublicIp(String ipAddress) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 72225763381..2fb549acc11 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -21,6 +21,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.ZoneEndpoint; import com.yahoo.config.provision.ZoneEndpoint.AccessType; @@ -161,8 +162,8 @@ public class MockNodeRepository extends NodeRepository { // Emulate host in tenant account nodes.add(Node.create("dockerhost2", ipConfig(101, 1, 3), "dockerhost2.yahoo.com", flavors.getFlavorOrThrow("large"), NodeType.host) - .wireguardPubKey(WireguardKey.from("000011112222333344445555666677778888999900c=")) - .wireguardKeyTimestamp(Instant.ofEpochMilli(123L)) + .wireguardKey(new WireguardKeyWithTimestamp(WireguardKey.from("000011112222333344445555666677778888999900c="), + Instant.ofEpochMilli(123L))) .cloudAccount(tenantAccount).build()); nodes.add(Node.create("dockerhost3", ipConfig(102, 1, 3), "dockerhost3.yahoo.com", flavors.getFlavorOrThrow("large"), NodeType.host).cloudAccount(defaultCloudAccount).build()); @@ -176,8 +177,8 @@ public class MockNodeRepository extends NodeRepository { // Config servers nodes.add(Node.create("cfg1", ipConfig(201), "cfg1.yahoo.com", flavors.getFlavorOrThrow("default"), NodeType.config) .cloudAccount(defaultCloudAccount) - .wireguardPubKey(WireguardKey.from("lololololololololololololololololololololoo=")) - .wireguardKeyTimestamp(Instant.ofEpochMilli(456L)) + .wireguardKey(new WireguardKeyWithTimestamp(WireguardKey.from("lololololololololololololololololololololoo="), + Instant.ofEpochMilli(456L))) .build()); nodes.add(Node.create("cfg2", ipConfig(202), "cfg2.yahoo.com", flavors.getFlavorOrThrow("default"), NodeType.config) .cloudAccount(defaultCloudAccount) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json index 928e91861a2..54a0e7e9757 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json @@ -119,6 +119,10 @@ ], "additionalIpAddresses": [], "cloudAccount": "aws:111222333444", - "wireguardPubkey":"lololololololololololololololololololololoo=", + "wireguard": { + "key": "lololololololololololololololololololololoo=", + "timestamp": 456 + }, + "wireguardPubkey": "lololololololololololololololololololololoo=", "wireguardKeyTimestamp": 456 } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json index 72b5483d849..d3f1a8082ae 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json @@ -117,6 +117,10 @@ "ipAddresses": ["127.0.101.1", "::101:1"], "additionalIpAddresses": ["::101:2", "::101:3", "::101:4"], "cloudAccount": "aws:777888999000", + "wireguard": { + "key": "000011112222333344445555666677778888999900c=", + "timestamp": 123 + }, "wireguardPubkey": "000011112222333344445555666677778888999900c=", "wireguardKeyTimestamp": 123 } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json index d0d6df71fc1..404cf9a9a80 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json @@ -118,6 +118,10 @@ "ipAddresses": ["127.0.4.1", "::4:1"], "additionalIpAddresses": [], "cloudAccount": "aws:111222333444", + "wireguard": { + "key": "lololololololololololololololololololololoo=", + "timestamp": 123 + }, "wireguardPubkey": "lololololololololololololololololololololoo=", "wireguardKeyTimestamp": 123 } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json index 7bee06adc87..8e9af7f680f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json @@ -4,7 +4,11 @@ "hostname": "cfg1.yahoo.com", "wireguardPubkey": "lololololololololololololololololololololoo=", "wireguardKeyTimestamp":456, - "ipAddresses": ["::201:1"] + "ipAddresses": ["::201:1"], + "wireguard": { + "key": "lololololololololololololololololololololoo=", + "timestamp": 456 + } } ] } diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 575a52d01fb..d098bdde8b6 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -711,7 +711,7 @@ void Test::requireThatQueryGluesEverythingTogether() { EXPECT_EQUAL(1u, md->getNumTermFields()); query.optimize(); - query.fetchPostings(); + query.fetchPostings(requestContext.getDoom()); SearchIterator::UP search = query.createSearch(*md); ASSERT_TRUE(search.get()); } @@ -744,7 +744,7 @@ void checkQueryAddsLocation(const string &loc_in, const string &loc_out) { MatchData::UP md = mdl.createMatchData(); EXPECT_EQUAL(2u, md->getNumTermFields()); - query.fetchPostings(); + query.fetchPostings(requestContext.getDoom()); SearchIterator::UP search = query.createSearch(*md); ASSERT_TRUE(search.get()); if (!EXPECT_NOT_EQUAL(string::npos, search->asString().find(loc_out))) { @@ -966,7 +966,7 @@ Test::requireThatWhiteListBlueprintCanBeUsed() MatchData::UP md = mdl.createMatchData(); query.optimize(); - query.fetchPostings(); + query.fetchPostings(requestContext.getDoom()); SearchIterator::UP search = query.createSearch(*md); SimpleResult exp = SimpleResult().addHit(1).addHit(5).addHit(7).addHit(11); SimpleResult act; diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp index 1528b327747..b8027bff04a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/queryeval/searchable.h> #include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/searchlib/queryeval/irequestcontext.h> #include <vespa/searchlib/query/tree/range.h> #include <vespa/searchlib/query/tree/simplequery.h> @@ -98,7 +99,7 @@ AttributeLimiter::create_search(size_t want_hits, size_t max_group_size, bool st FieldSpecList field; // single field API is protected field.add(FieldSpec(_attribute_name, my_field_id, my_handle)); _blueprint = _searchable_attributes.createBlueprint(_requestContext, field, node); - _blueprint->fetchPostings(ExecuteInfo::create(strictSearch)); + _blueprint->fetchPostings(ExecuteInfo::create(strictSearch, &_requestContext.getDoom())); _estimatedHits.store(_blueprint->getState().estimate().estHits, std::memory_order_relaxed); _blueprint->freeze(); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index 5ae671b88cb..758ef35ebc9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -201,9 +201,9 @@ MatchToolsFactory(QueryLimiter & queryLimiter, trace.addEvent(5, "Optimize query execution plan"); _query.optimize(); trace.addEvent(4, "Perform dictionary lookups and posting lists initialization"); - _query.fetchPostings(); + _query.fetchPostings(_requestContext.getDoom()); if (is_search) { - _query.handle_global_filter(searchContext.getDocIdLimit(), + _query.handle_global_filter(_requestContext.getDoom(), searchContext.getDocIdLimit(), _attribute_blueprint_params.global_filter_lower_limit, _attribute_blueprint_params.global_filter_upper_limit, thread_bundle, trace); diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index d0738f1857f..22f6ec9cc88 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -247,13 +247,14 @@ Query::optimize() } void -Query::fetchPostings() +Query::fetchPostings(const vespalib::Doom & doom) { - _blueprint->fetchPostings(search::queryeval::ExecuteInfo::create(true, 1.0)); + _blueprint->fetchPostings(search::queryeval::ExecuteInfo::create(true, &doom)); } void -Query::handle_global_filter(uint32_t docid_limit, double global_filter_lower_limit, double global_filter_upper_limit, +Query::handle_global_filter(const vespalib::Doom & doom, uint32_t docid_limit, + double global_filter_lower_limit, double global_filter_upper_limit, vespalib::ThreadBundle &thread_bundle, search::engine::Trace& trace) { if (!handle_global_filter(*_blueprint, docid_limit, global_filter_lower_limit, global_filter_upper_limit, thread_bundle, &trace)) { @@ -264,7 +265,7 @@ Query::handle_global_filter(uint32_t docid_limit, double global_filter_lower_lim _blueprint = Blueprint::optimize(std::move(_blueprint)); LOG(debug, "blueprint after handle_global_filter:\n%s\n", _blueprint->asString().c_str()); // strictness may change if optimized order changed: - fetchPostings(); + fetchPostings(doom); } bool diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index b0299307e92..1a3136042a7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -98,9 +98,10 @@ public: * test to verify the original query without optimization. **/ void optimize(); - void fetchPostings(); + void fetchPostings(const vespalib::Doom & doom); - void handle_global_filter(uint32_t docid_limit, double global_filter_lower_limit, double global_filter_upper_limit, + void handle_global_filter(const vespalib::Doom & doom, uint32_t docid_limit, + double global_filter_lower_limit, double global_filter_upper_limit, vespalib::ThreadBundle &thread_bundle, search::engine::Trace& trace); /** diff --git a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp index c7bd0e917f3..c2a39779061 100644 --- a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp +++ b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/fuzzy/levenshtein_dfa.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/text/utf8.h> #include <filesystem> #include <fstream> #include <iostream> @@ -26,13 +27,24 @@ using namespace search::attribute; using namespace search; using vespalib::FuzzyMatcher; using vespalib::datastore::AtomicEntryRef; +using vespalib::datastore::EntryRef; using vespalib::fuzzy::LevenshteinDfa; +using vespalib::Utf8Reader; +using vespalib::Utf8Writer; using StringEnumStore = EnumStoreT<const char*>; using DictionaryEntry = std::pair<std::string, size_t>; using RawDictionary = std::vector<DictionaryEntry>; using StringVector = std::vector<std::string>; +namespace { + +const char* char_from_u8(const char8_t* p) { + return reinterpret_cast<const char*>(p); +} + +} + RawDictionary read_dictionary() { @@ -109,11 +121,11 @@ struct MatchStats { template <bool collect_matches> void -brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, MatchStats& stats, StringVector& matched_words) +brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) { auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); vespalib::Timer timer; - FuzzyMatcher matcher(target, 2, 0, false); + FuzzyMatcher matcher(target, 2, prefix_size, cased); auto itr = view.begin(); size_t matches = 0; size_t seeks = 0; @@ -133,15 +145,33 @@ brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumS template <bool collect_matches> void -dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, MatchStats& stats, StringVector& matched_words) +dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) { auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); vespalib::Timer timer; - DfaFuzzyMatcher matcher(target, 2, false, LevenshteinDfa::DfaType::Explicit); - auto itr = view.begin(); + DfaFuzzyMatcher matcher(target, 2, prefix_size, cased, LevenshteinDfa::DfaType::Explicit); + Utf8Reader reader(vespalib::stringref(target.data(), target.size())); + std::string target_copy; + Utf8Writer<std::string> writer(target_copy); + for (size_t pos = 0; pos < prefix_size && reader.hasMore(); ++pos) { + auto code_point = reader.getChar(); + writer.putChar(code_point); + } + auto prefix_cmp = store.make_folded_comparator_prefix(target_copy.c_str()); + auto itr = prefix_size > 0 ? view.lowerBound(AtomicEntryRef(), prefix_cmp) : view.begin(); + auto itr_end = itr; + if (itr_end.valid()) { + if (prefix_size > 0) { + if (!prefix_cmp.less(EntryRef(), itr_end.getKey().load_relaxed())) { + itr_end.seekPast(AtomicEntryRef(), prefix_cmp); + } + } else { + itr_end.end(); + } + } size_t matches = 0; size_t seeks = 0; - while (itr.valid()) { + while (itr != itr_end) { auto word = store.get_value(itr.getKey().load_relaxed()); if (matcher.is_match(word, itr, store.get_data_store())) { ++itr; @@ -156,10 +186,58 @@ dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& st stats.add_sample(matches, seeks, timer.elapsed()); } -struct DfaFuzzyMatcherTest : public ::testing::Test { +template <bool collect_matches> +void +dfa_fuzzy_match_in_dictionary_no_skip(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) +{ + auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); + vespalib::Timer timer; + DfaFuzzyMatcher matcher(target, 2, prefix_size, cased, LevenshteinDfa::DfaType::Explicit); + auto itr = view.begin(); + size_t matches = 0; + size_t seeks = 0; + for (;itr.valid(); ++itr) { + auto word = store.get_value(itr.getKey().load_relaxed()); + if (matcher.is_match(word)) { + ++matches; + if (collect_matches) { + matched_words.push_back(word); + } + } else { + ++seeks; + } + } + stats.add_sample(matches, seeks, timer.elapsed()); +} + +struct TestParam +{ + vespalib::string _name; + bool _cased; + + TestParam(vespalib::string name, bool cased) + : _name(std::move(name)), + _cased(cased) + { + } + TestParam(const TestParam&); + ~TestParam(); +}; + +TestParam::TestParam(const TestParam&) = default; + +TestParam::~TestParam() = default; + +std::ostream& operator<<(std::ostream& os, const TestParam& param) +{ + os << param._name; + return os; +} + +struct DfaFuzzyMatcherTest : public ::testing::TestWithParam<TestParam> { StringEnumStore store; DfaFuzzyMatcherTest() - : store(true, DictionaryConfig(DictionaryConfig::Type::BTREE, DictionaryConfig::Match::UNCASED)) + : store(true, DictionaryConfig(DictionaryConfig::Type::BTREE, GetParam()._cased ? DictionaryConfig::Match::CASED : DictionaryConfig::Match::UNCASED)) {} void populate_dictionary(const StringVector& words) { auto updater = store.make_batch_updater(); @@ -170,18 +248,31 @@ struct DfaFuzzyMatcherTest : public ::testing::Test { updater.commit(); store.freeze_dictionary(); } - void expect_matches(std::string_view target, const StringVector& exp_matches) { + void expect_prefix_matches(std::string_view target, uint32_t prefix_size, const StringVector& exp_matches) { MatchStats stats; StringVector brute_force_matches; StringVector dfa_matches; - brute_force_fuzzy_match_in_dictionary<true>(target, store, stats, brute_force_matches); - dfa_fuzzy_match_in_dictionary<true>(target, store, stats, dfa_matches); + StringVector dfa_no_skip_matches; + bool cased = GetParam()._cased; + SCOPED_TRACE(target); + brute_force_fuzzy_match_in_dictionary<true>(target, store, prefix_size, cased, stats, brute_force_matches); + dfa_fuzzy_match_in_dictionary<true>(target, store, prefix_size, cased, stats, dfa_matches); + dfa_fuzzy_match_in_dictionary_no_skip<true>(target, store, prefix_size, cased, stats, dfa_no_skip_matches); EXPECT_EQ(exp_matches, brute_force_matches); EXPECT_EQ(exp_matches, dfa_matches); + EXPECT_EQ(exp_matches, dfa_no_skip_matches); + } + void expect_matches(std::string_view target, const StringVector& exp_matches) { + expect_prefix_matches(target, 0, exp_matches); } }; -TEST_F(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) +INSTANTIATE_TEST_SUITE_P(DfaFuzzyMatcherMultiTest, + DfaFuzzyMatcherTest, + testing::Values(TestParam("uncased", false), TestParam("cased", true)), + testing::PrintToStringParamName()); + +TEST_P(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) { StringVector words = { "board", "boat", "bob", "door", "food", "foot", "football", "foothill", "for", "forbid", "force", "ford", "forearm", "forecast", "forest" }; @@ -194,23 +285,67 @@ TEST_F(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) expect_matches("forcecast", {"forecast"}); } +TEST_P(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary_with_prefix_size) +{ + bool cased = GetParam()._cased; + StringVector words = { "board", "boat", "bob", "door", "food", "foot", "football", "foothill", + "for", "forbid", "force", "ford", "forearm", "forecast", "forest", "H", "HA", "h", "ha", char_from_u8(u8"Ørn"), char_from_u8(u8"øre"), char_from_u8(u8"Ås"), char_from_u8(u8"ås")}; + populate_dictionary(words); + expect_prefix_matches("a", 1, {}); + expect_prefix_matches("b", 1, {"bob"}); + expect_prefix_matches("board", 1, {"board", "boat"}); + expect_prefix_matches("c", 1, {}); + expect_prefix_matches("food", 1, {"food", "foot", "for", "ford"}); + expect_prefix_matches("food", 2, {"food", "foot", "for", "ford"}); + expect_prefix_matches("food", 3, {"food", "foot"}); + expect_prefix_matches("foothill", 1, {"football", "foothill"}); + expect_prefix_matches("for", 1, {"food", "foot", "for", "force", "ford"}); + expect_prefix_matches("for", 2, {"food", "foot", "for", "force", "ford"}); + expect_prefix_matches("for", 3, {"for", "force", "ford"}); + expect_prefix_matches("force", 1, {"for", "force", "ford"}); + expect_prefix_matches("forcecast", 1, {"forecast"}); + expect_prefix_matches("forcecast", 4, {}); + expect_prefix_matches("z", 1, {}); + if (cased) { + expect_prefix_matches("h", 1, {"h", "ha"}); + expect_prefix_matches(char_from_u8(u8"Ø"), 1, {char_from_u8(u8"Ørn")}); + expect_prefix_matches(char_from_u8(u8"ø"), 1, {char_from_u8(u8"øre")}); + expect_prefix_matches(char_from_u8(u8"å"), 1, {char_from_u8(u8"ås")}); + /* Corner case: prefix length > target length means exact match */ + expect_prefix_matches("h", 2, {"h"}); + } else { + expect_prefix_matches("h", 1, {"H", "h", "HA", "ha"}); + expect_prefix_matches(char_from_u8(u8"ø"), 1, {char_from_u8(u8"øre"), char_from_u8(u8"Ørn")}); + expect_prefix_matches(char_from_u8(u8"å"), 1, {char_from_u8(u8"Ås"), char_from_u8(u8"ås")}); + /* Corner case: prefix length > target length means exact match */ + expect_prefix_matches("h", 2, {"H", "h"}); + } +} + void -benchmark_fuzzy_match_in_dictionary(const StringEnumStore& store, const RawDictionary& dict, size_t words_to_match, bool dfa_algorithm) +benchmark_fuzzy_match_in_dictionary(const StringEnumStore& store, const RawDictionary& dict, size_t words_to_match, bool cased, bool dfa_algorithm) { MatchStats stats; StringVector dummy; for (size_t i = 0; i < std::min(words_to_match, dict.size()); ++i) { const auto& entry = dict[i]; if (dfa_algorithm) { - dfa_fuzzy_match_in_dictionary<false>(entry.first, store, stats, dummy); + dfa_fuzzy_match_in_dictionary<false>(entry.first, store, 0, cased, stats, dummy); } else { - brute_force_fuzzy_match_in_dictionary<false>(entry.first, store, stats, dummy); + brute_force_fuzzy_match_in_dictionary<false>(entry.first, store, 0, cased, stats, dummy); } } std::cout << (dfa_algorithm ? "DFA:" : "Brute force:") << " samples=" << stats.samples << ", avg_matches=" << stats.avg_matches() << ", avg_seeks=" << stats.avg_seeks() << ", avg_elapsed_ms=" << stats.avg_elapsed_ms() << std::endl; } -TEST_F(DfaFuzzyMatcherTest, benchmark_fuzzy_match_in_dictionary) +using DfaFuzzyMatcherBenchmarkTest = DfaFuzzyMatcherTest; + +INSTANTIATE_TEST_SUITE_P(DfaFuzzyMatcherBenchmarkMultiTest, + DfaFuzzyMatcherBenchmarkTest, + testing::Values(TestParam("uncased", false)), + testing::PrintToStringParamName()); + +TEST_P(DfaFuzzyMatcherBenchmarkTest, benchmark_fuzzy_match_in_dictionary) { if (!benchmarking_enabled()) { GTEST_SKIP() << "benchmarking not enabled"; @@ -219,8 +354,9 @@ TEST_F(DfaFuzzyMatcherTest, benchmark_fuzzy_match_in_dictionary) populate_dictionary(to_string_vector(dict)); std::cout << "Unique words: " << store.get_num_uniques() << std::endl; sort_by_freq(dict); - benchmark_fuzzy_match_in_dictionary(store, dict, dfa_words_to_match, true); - benchmark_fuzzy_match_in_dictionary(store, dict, brute_force_words_to_match, false); + bool cased = GetParam()._cased; + benchmark_fuzzy_match_in_dictionary(store, dict, dfa_words_to_match, cased, true); + benchmark_fuzzy_match_in_dictionary(store, dict, brute_force_words_to_match, cased, false); } int diff --git a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp index d7a854e0afc..6c6f05fd5e2 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp @@ -29,14 +29,14 @@ using namespace search::attribute::test; namespace { void -setupAttributeManager(MockAttributeManager &manager) +setupAttributeManager(MockAttributeManager &manager, bool isFilter) { AttributeVector::DocId docId; { - AttributeVector::SP attr_sp = AttributeFactory::createAttribute("integer", Config(BasicType("int64"))); + AttributeVector::SP attr_sp = AttributeFactory::createAttribute("integer", Config(BasicType("int64")).setIsFilter(isFilter)); manager.addAttribute(attr_sp); - IntegerAttribute *attr = (IntegerAttribute*)(attr_sp.get()); + auto *attr = (IntegerAttribute*)(attr_sp.get()); for (size_t i = 1; i < 10; ++i) { attr->addDoc(docId); assert(i == docId); @@ -45,10 +45,10 @@ setupAttributeManager(MockAttributeManager &manager) } } { - AttributeVector::SP attr_sp = AttributeFactory::createAttribute("string", Config(BasicType("string"))); + AttributeVector::SP attr_sp = AttributeFactory::createAttribute("string", Config(BasicType("string")).setIsFilter(isFilter)); manager.addAttribute(attr_sp); - StringAttribute *attr = (StringAttribute*)(attr_sp.get()); + auto *attr = (StringAttribute*)(attr_sp.get()); for (size_t i = 1; i < 10; ++i) { attr->addDoc(docId); assert(i == docId); @@ -58,9 +58,9 @@ setupAttributeManager(MockAttributeManager &manager) } { AttributeVector::SP attr_sp = AttributeFactory::createAttribute( - "multi", Config(BasicType("int64"), search::attribute::CollectionType("array"))); + "multi", Config(BasicType("int64"), search::attribute::CollectionType("array")).setIsFilter(isFilter)); manager.addAttribute(attr_sp); - IntegerAttribute *attr = (IntegerAttribute*)(attr_sp.get()); + auto *attr = (IntegerAttribute*)(attr_sp.get()); for (size_t i = 1; i < 10; ++i) { attr->addDoc(docId); assert(i == docId); @@ -78,35 +78,43 @@ struct WS { TermFieldHandle handle; std::vector<std::pair<std::string, uint32_t> > tokens; - WS(IAttributeManager & manager) : attribute_manager(manager), layout(), handle(layout.allocTermField(fieldId)), tokens() { + explicit WS(IAttributeManager & manager) + : attribute_manager(manager), + layout(), handle(layout.allocTermField(fieldId)), + tokens() + { MatchData::UP tmp = layout.createMatchData(); ASSERT_TRUE(tmp->resolveTermField(handle)->getFieldId() == fieldId); } WS &add(const std::string &token, uint32_t weight) { - tokens.push_back(std::make_pair(token, weight)); + tokens.emplace_back(token, weight); return *this; } Node::UP createNode() const { - SimpleWeightedSetTerm *node = new SimpleWeightedSetTerm(tokens.size(), "view", 0, Weight(0)); - for (size_t i = 0; i < tokens.size(); ++i) { - node->addTerm(tokens[i].first, Weight(tokens[i].second)); + auto *node = new SimpleWeightedSetTerm(tokens.size(), "view", 0, Weight(0)); + for (const auto & token : tokens) { + node->addTerm(token.first, Weight(token.second)); } return Node::UP(node); } - bool isGenericSearch(Searchable &searchable, const std::string &field, bool strict) const { + SearchIterator::UP + createSearch(Searchable &searchable, const std::string &field, bool strict) const { AttributeContext ac(attribute_manager); FakeRequestContext requestContext(&ac); MatchData::UP md = layout.createMatchData(); Node::UP node = createNode(); FieldSpecList fields; - fields.add(FieldSpec(field, fieldId, handle)); + fields.add(FieldSpec(field, fieldId, handle, ac.getAttribute(field)->getIsFilter())); queryeval::Blueprint::UP bp = searchable.createBlueprint(requestContext, fields, *node); bp->fetchPostings(queryeval::ExecuteInfo::create(strict)); SearchIterator::UP sb = bp->createSearch(*md, strict); - return (dynamic_cast<WeightedSetTermSearch*>(sb.get()) != 0); + return sb; + } + bool isWeightedSetTermSearch(Searchable &searchable, const std::string &field, bool strict) const { + return dynamic_cast<WeightedSetTermSearch *>(createSearch(searchable, field, strict).get()) != nullptr; } FakeResult search(Searchable &searchable, const std::string &field, bool strict) const { @@ -140,23 +148,58 @@ struct WS { } // namespace <unnamed> +void test_tokens(bool isFilter, const std::vector<uint32_t> & docs) { + MockAttributeManager manager; + setupAttributeManager(manager, isFilter); + AttributeBlueprintFactory adapter; + + FakeResult expect = FakeResult(); + WS ws = WS(manager); + for (uint32_t doc : docs) { + auto docS = vespalib::stringify(doc); + int32_t weight = doc * 10; + expect.doc(doc).weight(weight).pos(0); + ws.add(docS, weight); + } + + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "integer", true)); + EXPECT_TRUE(!ws.isWeightedSetTermSearch(adapter, "integer", false)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "string", true)); + EXPECT_TRUE(!ws.isWeightedSetTermSearch(adapter, "string", false)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", true)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", false)); + + EXPECT_EQUAL(expect, ws.search(adapter, "integer", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "integer", false)); + EXPECT_EQUAL(expect, ws.search(adapter, "string", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "string", false)); + EXPECT_EQUAL(expect, ws.search(adapter, "multi", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "multi", false)); +} TEST("attribute_weighted_set_test") { + test_tokens(false, {3, 5, 7}); + test_tokens(true, {3, 5, 7}); + test_tokens(false, {3}); +} + +TEST("attribute_weighted_set_single_token_filter_lifted_out") { MockAttributeManager manager; - setupAttributeManager(manager); + setupAttributeManager(manager, true); AttributeBlueprintFactory adapter; - FakeResult expect = FakeResult() - .doc(3).elem(0).weight(30).pos(0) - .doc(5).elem(0).weight(50).pos(0) - .doc(7).elem(0).weight(70).pos(0); - WS ws = WS(manager).add("7", 70).add("5", 50).add("3", 30); - - EXPECT_TRUE(ws.isGenericSearch(adapter, "integer", true)); - EXPECT_TRUE(!ws.isGenericSearch(adapter, "integer", false)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "string", true)); - EXPECT_TRUE(!ws.isGenericSearch(adapter, "string", false)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "multi", true)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "multi", false)); + FakeResult expect = FakeResult().doc(3).elem(0).weight(30).pos(0); + WS ws = WS(manager).add("3", 30); + + EXPECT_EQUAL("search::FilterAttributeIteratorStrict<search::attribute::SingleNumericSearchContext<long, search::attribute::NumericMatcher<long> > >", + ws.createSearch(adapter, "integer", true)->getClassName()); + EXPECT_EQUAL("search::FilterAttributeIteratorT<search::attribute::SingleNumericSearchContext<long, search::attribute::NumericMatcher<long> > >", + ws.createSearch(adapter, "integer", false)->getClassName()); + EXPECT_EQUAL("search::FilterAttributeIteratorStrict<search::attribute::SingleEnumSearchContext<char const*, search::attribute::StringSearchContext> >", + ws.createSearch(adapter, "string", true)->getClassName()); + EXPECT_EQUAL("search::FilterAttributeIteratorT<search::attribute::SingleEnumSearchContext<char const*, search::attribute::StringSearchContext> >", + ws.createSearch(adapter, "string", false)->getClassName()); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", true)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", false)); EXPECT_EQUAL(expect, ws.search(adapter, "integer", true)); EXPECT_EQUAL(expect, ws.search(adapter, "integer", false)); diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index ef0fd56840a..c617db871a7 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -128,9 +128,9 @@ TEST("test And propagates updated histestimate") { const RememberExecuteInfo & child = dynamic_cast<const RememberExecuteInfo &>(bp.getChild(i)); EXPECT_EQUAL((i == 0), child.executeInfo.isStrict()); } - EXPECT_EQUAL(1.0, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(0)).executeInfo.hitRate()); - EXPECT_EQUAL(1.0/250, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(1)).executeInfo.hitRate()); - EXPECT_EQUAL(1.0/(250*25), dynamic_cast<const RememberExecuteInfo &>(bp.getChild(2)).executeInfo.hitRate()); + EXPECT_EQUAL(1.0f, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(0)).executeInfo.hitRate()); + EXPECT_EQUAL(1.0f/250, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(1)).executeInfo.hitRate()); + EXPECT_EQUAL(1.0f/(250*25), dynamic_cast<const RememberExecuteInfo &>(bp.getChild(2)).executeInfo.hitRate()); } TEST("test And Blueprint") { diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 1519bb14554..b4cdd621b71 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -337,10 +337,7 @@ public: if (tfmda.size() == 1) { // search in exactly one field fef::TermFieldMatchData &tfmd = *tfmda[0]; - return search::common::create_location_iterator(tfmd, - _attribute.getNumDocs(), - strict, - _location); + return common::create_location_iterator(tfmd, _attribute.getNumDocs(), strict, _location); } else { LOG(debug, "wrong size tfmda: %zu (fallback to old location iterator)\n", tfmda.size()); } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp index 108128eeb39..94c560a0dae 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp @@ -30,7 +30,7 @@ protected: const attribute::IAttributeVector &attribute() const { return _attr; } public: - UseAttr(const attribute::IAttributeVector & attr) + explicit UseAttr(const attribute::IAttributeVector & attr) : _attr(attr) {} }; @@ -40,7 +40,7 @@ class UseStringEnum : public UseAttr { public: using TokenT = uint32_t; - UseStringEnum(const IAttributeVector & attr) + explicit UseStringEnum(const IAttributeVector & attr) : UseAttr(attr) {} auto mapToken(const ISearchContext &context) const { return attribute().findFoldedEnums(context.queryTerm()->getTerm()); @@ -56,7 +56,7 @@ class UseInteger : public UseAttr { public: using TokenT = uint64_t; - UseInteger(const IAttributeVector & attr) : UseAttr(attr) {} + explicit UseInteger(const IAttributeVector & attr) : UseAttr(attr) {} std::vector<int64_t> mapToken(const ISearchContext &context) const { std::vector<int64_t> result; Int64Range range(context.getAsIntegerTerm()); @@ -157,6 +157,10 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr assert(tfmda.size() == 1); assert(getState().numFields() == 1); fef::TermFieldMatchData &tfmd = *tfmda[0]; + bool field_is_filter = getState().fields()[0].isFilter(); + if (field_is_filter && (_contexts.size() == 1)) { + return _contexts[0]->createIterator(&tfmd, strict); + } if (strict) { // use generic weighted set search fef::MatchDataLayout layout; auto handle = layout.allocTermField(tfmd.getFieldId()); @@ -167,7 +171,6 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr // TODO: pass ownership with unique_ptr children[i] = _contexts[i]->createIterator(child_tfmd, true).release(); } - bool field_is_filter = getState().fields()[0].isFilter(); return queryeval::WeightedSetTermSearch::create(children, tfmd, field_is_filter, _weights, std::move(match_data)); } else { // use attribute filter optimization bool isString = (_attr.isStringType() && _attr.hasEnum()); @@ -182,18 +185,16 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr } queryeval::SearchIterator::UP -AttributeWeightedSetBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +AttributeWeightedSetBlueprint::createFilterSearch(bool strict, FilterConstraint) const { - (void) constraint; std::vector<std::unique_ptr<queryeval::SearchIterator>> children; children.reserve(_contexts.size()); for (auto& context : _contexts) { - auto wrapper = std::make_unique<search::queryeval::FilterWrapper>(1); + auto wrapper = std::make_unique<queryeval::FilterWrapper>(1); wrapper->wrap(context->createIterator(wrapper->tfmda()[0], strict)); children.emplace_back(std::move(wrapper)); } - search::queryeval::UnpackInfo unpack_info; - return search::queryeval::OrSearch::create(std::move(children), strict, unpack_info); + return queryeval::OrSearch::create(std::move(children), strict, queryeval::UnpackInfo()); } void diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp index 580c34bd5d0..b16fdc12a9a 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp +++ b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp @@ -1,17 +1,98 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dfa_fuzzy_matcher.h" +#include <vespa/vespalib/text/utf8.h> +#include <vespa/vespalib/text/lowercase.h> using vespalib::fuzzy::LevenshteinDfa; +using vespalib::LowerCase; +using vespalib::Utf8Reader; +using vespalib::Utf8ReaderForZTS; namespace search::attribute { -DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, bool cased, LevenshteinDfa::DfaType dfa_type) - : _dfa(vespalib::fuzzy::LevenshteinDfa::build(target, max_edits, (cased ? LevenshteinDfa::Casing::Cased : LevenshteinDfa::Casing::Uncased), dfa_type)), - _successor() +namespace { + +std::vector<uint32_t> +extract_prefix(std::string_view target, uint32_t prefix_size, bool cased) +{ + std::vector<uint32_t> result; + result.reserve(prefix_size); + Utf8Reader reader(vespalib::stringref(target.data(), target.size())); + for (size_t pos = 0; pos < prefix_size && reader.hasMore(); ++pos) { + uint32_t code_point = reader.getChar(); + if (!cased) { + code_point = LowerCase::convert(code_point); + } + result.emplace_back(code_point); + } + return result; +} + +std::string_view +extract_suffix(std::string_view target, uint32_t prefix_size) { + Utf8Reader reader(vespalib::stringref(target.data(), target.size())); + for (size_t pos = 0; pos < prefix_size && reader.hasMore(); ++pos) { + (void) reader.getChar(); + } + std::string_view result = target; + result.remove_prefix(reader.getPos()); + return result; +} + +} + +DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, LevenshteinDfa::DfaType dfa_type) + : _dfa(vespalib::fuzzy::LevenshteinDfa::build(extract_suffix(target, prefix_size), max_edits, (cased ? LevenshteinDfa::Casing::Cased : LevenshteinDfa::Casing::Uncased), dfa_type)), + _successor(), + _prefix(extract_prefix(target, prefix_size, cased)), + _successor_suffix(), + _prefix_size(prefix_size), + _cased(cased) +{ + _successor = _prefix; } DfaFuzzyMatcher::~DfaFuzzyMatcher() = default; +const char* +DfaFuzzyMatcher::skip_prefix(const char* word) const +{ + Utf8ReaderForZTS reader(word); + size_t pos = 0; + for (; pos < _prefix.size() && reader.hasMore(); ++pos) { + (void) reader.getChar(); + } + assert(pos == _prefix.size()); + return reader.get_current_ptr(); +} + +bool +DfaFuzzyMatcher::is_match(const char* word) const +{ + if (_prefix_size > 0) { + Utf8ReaderForZTS reader(word); + size_t pos = 0; + for (; pos < _prefix.size() && reader.hasMore(); ++pos) { + uint32_t code_point = reader.getChar(); + if (!_cased) { + code_point = LowerCase::convert(code_point); + } + if (code_point != _prefix[pos]) { + break; + } + } + if (!reader.hasMore() && pos == _prefix.size() && pos < _prefix_size) { + return true; + } + if (pos != _prefix_size) { + return false; + } + word = reader.get_current_ptr(); + } + auto match = _dfa.match(word); + return match.matches(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h index fcba13f85a4..7116b4d8662 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h +++ b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h @@ -5,6 +5,7 @@ #include "dfa_string_comparator.h" #include <vespa/vespalib/datastore/atomic_entry_ref.h> #include <vespa/vespalib/fuzzy/levenshtein_dfa.h> +#include <iostream> namespace search::attribute { @@ -17,22 +18,54 @@ namespace search::attribute { class DfaFuzzyMatcher { private: vespalib::fuzzy::LevenshteinDfa _dfa; - std::vector<uint32_t> _successor; + std::vector<uint32_t> _successor; + std::vector<uint32_t> _prefix; + std::vector<uint32_t> _successor_suffix; + uint32_t _prefix_size; + bool _cased; + const char* skip_prefix(const char* word) const; public: - DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, bool cased, vespalib::fuzzy::LevenshteinDfa::DfaType dfa_type); + DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, vespalib::fuzzy::LevenshteinDfa::DfaType dfa_type); ~DfaFuzzyMatcher(); + bool is_match(const char *word) const; + + /* + * If prefix size is nonzero then this variant of is_match() + * should only be called with words that starts with the extracted + * prefix of the target word. + * + * Caller must position iterator at right location using lower bound + * functionality in the dictionary. + */ template <typename DictionaryConstIteratorType> bool is_match(const char* word, DictionaryConstIteratorType& itr, const DfaStringComparator::DataStoreType& data_store) { - auto match = _dfa.match(word, _successor); - if (match.matches()) { - return true; + if (_prefix_size > 0) { + word = skip_prefix(word); + if (_prefix.size() < _prefix_size) { + if (*word == '\0') { + return true; + } + _successor.resize(_prefix.size()); + _successor.emplace_back(1); + } else { + auto match = _dfa.match(word, _successor_suffix); + if (match.matches()) { + return true; + } + _successor.resize(_prefix.size()); + _successor.insert(_successor.end(), _successor_suffix.begin(), _successor_suffix.end()); + } } else { - DfaStringComparator cmp(data_store, _successor); - itr.seek(vespalib::datastore::AtomicEntryRef(), cmp); - return false; + auto match = _dfa.match(word, _successor); + if (match.matches()) { + return true; + } } + DfaStringComparator cmp(data_store, _successor); + itr.seek(vespalib::datastore::AtomicEntryRef(), cmp); + return false; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp index 3c0bae00047..e2566c94f1c 100644 --- a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp +++ b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp @@ -11,8 +11,8 @@ class DocumentWeightOrFilterSearchImpl : public DocumentWeightOrFilterSearch { AttributeIteratorPack _children; public: - DocumentWeightOrFilterSearchImpl(AttributeIteratorPack&& children); - ~DocumentWeightOrFilterSearchImpl(); + explicit DocumentWeightOrFilterSearchImpl(AttributeIteratorPack&& children); + ~DocumentWeightOrFilterSearchImpl() override; void doSeek(uint32_t docId) override; @@ -67,7 +67,7 @@ DocumentWeightOrFilterSearchImpl::doUnpack(uint32_t) { } -std::unique_ptr<search::queryeval::SearchIterator> +std::unique_ptr<queryeval::SearchIterator> DocumentWeightOrFilterSearch::create(std::vector<DocumentWeightIterator>&& children) { if (children.empty()) { diff --git a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h index 62be883ab52..c601856573f 100644 --- a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h +++ b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h @@ -9,15 +9,15 @@ namespace search::attribute { * Filter iterator on top of document weight iterators with OR semantics used during * calculation of global filter for weighted set terms, wand terms and dot product terms. */ -class DocumentWeightOrFilterSearch : public search::queryeval::SearchIterator +class DocumentWeightOrFilterSearch : public queryeval::SearchIterator { protected: DocumentWeightOrFilterSearch() - : search::queryeval::SearchIterator() + : queryeval::SearchIterator() { } public: - static std::unique_ptr<search::queryeval::SearchIterator> create(std::vector<DocumentWeightIterator>&& children); + static std::unique_ptr<queryeval::SearchIterator> create(std::vector<DocumentWeightIterator>&& children); }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 3f6085ef7ff..94d1a4917fd 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -621,7 +621,7 @@ IntermediateBlueprint::fetchPostings(const ExecuteInfo &execInfo) double nextHitRate = execInfo.hitRate(); for (size_t i = 0; i < _children.size(); ++i) { Blueprint & child = *_children[i]; - child.fetchPostings(ExecuteInfo::create(execInfo.isStrict() && inheritStrict(i), nextHitRate)); + child.fetchPostings(ExecuteInfo::create(execInfo.isStrict() && inheritStrict(i), nextHitRate, execInfo.getDoom())); nextHitRate = computeNextHitRate(child, nextHitRate); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp index 795f5f1424a..4322cafb5c8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp @@ -66,7 +66,7 @@ DotProductBlueprint::createFilterSearch(bool strict, FilterConstraint constraint void DotProductBlueprint::fetchPostings(const ExecuteInfo &execInfo) { - ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo.hitRate()); + ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo); for (size_t i = 0; i < _terms.size(); ++i) { _terms[i]->fetchPostings(childInfo); } diff --git a/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp b/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp index 604e20d2262..e5d20f047f5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp @@ -4,17 +4,7 @@ namespace search::queryeval { -const ExecuteInfo ExecuteInfo::TRUE(true, 1.0); -const ExecuteInfo ExecuteInfo::FALSE(false, 1.0); - -ExecuteInfo -ExecuteInfo::create(bool strict) { - return create(strict, 1.0); -} - -ExecuteInfo -ExecuteInfo::create(bool strict, double hitRate) { - return ExecuteInfo(strict, hitRate); -} +const ExecuteInfo ExecuteInfo::TRUE(true, 1.0, nullptr); +const ExecuteInfo ExecuteInfo::FALSE(false, 1.0, nullptr); } diff --git a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h index e161b2bdab7..2dd34284bef 100644 --- a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h +++ b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright 2019 Oath inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/vespalib/util/doom.h> + namespace search::queryeval { /** @@ -11,20 +12,37 @@ namespace search::queryeval { */ class ExecuteInfo { public: - ExecuteInfo() : ExecuteInfo(false, 1.0) { } - bool isStrict() const { return _strict; } - double hitRate() const { return _hitRate; } + ExecuteInfo() noexcept : ExecuteInfo(false, 1.0F, nullptr) { } + bool isStrict() const noexcept { return _strict; } + float hitRate() const noexcept { return _hitRate; } + bool soft_doom() const noexcept { return _doom && _doom->soft_doom(); } + const vespalib::Doom * getDoom() const { return _doom; } static const ExecuteInfo TRUE; static const ExecuteInfo FALSE; - static ExecuteInfo create(bool strict); - static ExecuteInfo create(bool strict, double HitRate); + static ExecuteInfo create(bool strict, const ExecuteInfo & org) noexcept { + return {strict, org._hitRate, org.getDoom()}; + } + static ExecuteInfo create(bool strict, const vespalib::Doom * doom) noexcept { + return create(strict, 1.0F, doom); + } + static ExecuteInfo create(bool strict, float hitRate, const vespalib::Doom * doom) noexcept { + return {strict, hitRate, doom}; + } + static ExecuteInfo create(bool strict) noexcept { + return create(strict, 1.0F); + } + static ExecuteInfo create(bool strict, float hitRate) noexcept { + return create(strict, hitRate, nullptr); + } private: - ExecuteInfo(bool strict, double hitRate_in) - : _hitRate(hitRate_in), + ExecuteInfo(bool strict, float hitRate_in, const vespalib::Doom * doom) noexcept + : _doom(doom), + _hitRate(hitRate_in), _strict(strict) { } - double _hitRate; - bool _strict; + const vespalib::Doom * _doom; + float _hitRate; + bool _strict; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp index 9c3910b20f9..16461487525 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp @@ -57,7 +57,7 @@ void SameElementBlueprint::fetchPostings(const ExecuteInfo &execInfo) { for (size_t i = 0; i < _terms.size(); ++i) { - _terms[i]->fetchPostings(ExecuteInfo::create(execInfo.isStrict() && (i == 0), execInfo.hitRate())); + _terms[i]->fetchPostings(ExecuteInfo::create(execInfo.isStrict() && (i == 0), execInfo)); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp index 48a09f099a6..eb6241a99d5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp @@ -4,7 +4,6 @@ #include "wand_parts.h" #include "parallel_weak_and_search.h" #include <vespa/searchlib/queryeval/field_spec.hpp> -#include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.hpp> @@ -77,10 +76,10 @@ ParallelWeakAndBlueprint::createLeafSearch(const search::fef::TermFieldMatchData const State &childState = _terms[i]->getState(); assert(childState.numFields() == 1); // TODO: pass ownership with unique_ptr - terms.push_back(wand::Term(_terms[i]->createSearch(*childrenMatchData, true).release(), - _weights[i], - childState.estimate().estHits, - childState.field(0).resolve(*childrenMatchData))); + terms.emplace_back(_terms[i]->createSearch(*childrenMatchData, true).release(), + _weights[i], + childState.estimate().estHits, + childState.field(0).resolve(*childrenMatchData)); } return SearchIterator::UP (ParallelWeakAndSearch::create(terms, @@ -101,9 +100,9 @@ ParallelWeakAndBlueprint::createFilterSearch(bool strict, FilterConstraint const void ParallelWeakAndBlueprint::fetchPostings(const ExecuteInfo & execInfo) { - ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo.hitRate()); - for (size_t i = 0; i < _terms.size(); ++i) { - _terms[i]->fetchPostings(childInfo); + ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo); + for (const auto & _term : _terms) { + _term->fetchPostings(childInfo); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp index 4e06f170253..97f6bc2e6f8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp @@ -33,10 +33,8 @@ WeightedSetTermMatchingElementsSearch::WeightedSetTermMatchingElementsSearch(con _search() { _tfmda.add(&_tfmd); - auto generic_search = bp.createLeafSearch(_tfmda, false); - auto weighted_set_term_search = dynamic_cast<WeightedSetTermSearch *>(generic_search.get()); - generic_search.release(); - _search.reset(weighted_set_term_search); + _search.reset(static_cast<WeightedSetTermSearch *>(bp.createLeafSearch(_tfmda, false).release())); + } WeightedSetTermMatchingElementsSearch::~WeightedSetTermMatchingElementsSearch() = default; @@ -120,16 +118,16 @@ WeightedSetTermBlueprint::create_matching_elements_search(const MatchingElements if (fields.has_field(_children_field.getName())) { return std::make_unique<WeightedSetTermMatchingElementsSearch>(*this, _children_field.getName(), _terms); } else { - return std::unique_ptr<MatchingElementsSearch>(); + return {}; } } void WeightedSetTermBlueprint::fetchPostings(const ExecuteInfo &execInfo) { - ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo.hitRate()); - for (size_t i = 0; i < _terms.size(); ++i) { - _terms[i]->fetchPostings(childInfo); + ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo); + for (const auto & _term : _terms) { + _term->fetchPostings(childInfo); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h index 0e3c82444d7..9c8d6d88329 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h @@ -18,7 +18,7 @@ class WeightedSetTermBlueprint : public ComplexLeafBlueprint std::vector<Blueprint::UP> _terms; public: - WeightedSetTermBlueprint(const FieldSpec &field); + explicit WeightedSetTermBlueprint(const FieldSpec &field); WeightedSetTermBlueprint(const WeightedSetTermBlueprint &) = delete; WeightedSetTermBlueprint &operator=(const WeightedSetTermBlueprint &) = delete; ~WeightedSetTermBlueprint() override; diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp index ee3978705cf..8478a0d3c35 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp @@ -21,7 +21,7 @@ private: struct CmpDocId { const uint32_t *termPos; - CmpDocId(const uint32_t *tp) : termPos(tp) {} + explicit CmpDocId(const uint32_t *tp) : termPos(tp) {} bool operator()(const ref_t &a, const ref_t &b) const { return (termPos[a] < termPos[b]); } @@ -29,7 +29,7 @@ private: struct CmpWeight { const int32_t *weight; - CmpWeight(const int32_t *w) : weight(w) {} + explicit CmpWeight(const int32_t *w) : weight(w) {} bool operator()(const ref_t &a, const ref_t &b) const { return (weight[a] > weight[b]); } @@ -61,7 +61,7 @@ private: } public: - WeightedSetTermSearchImpl(search::fef::TermFieldMatchData &tmd, + WeightedSetTermSearchImpl(fef::TermFieldMatchData &tmd, bool field_is_filter, const std::vector<int32_t> &weights, IteratorPack &&iteratorPack) @@ -180,7 +180,7 @@ WeightedSetTermSearch::create(const std::vector<SearchIterator *> &children, //----------------------------------------------------------------------------- SearchIterator::UP -WeightedSetTermSearch::create(search::fef::TermFieldMatchData &tmd, +WeightedSetTermSearch::create(fef::TermFieldMatchData &tmd, bool field_is_filter, const std::vector<int32_t> &weights, std::vector<DocumentWeightIterator> &&iterators) diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 2972ea7745e..1f6a15c7c8f 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -25,7 +25,7 @@ com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2 com.github.luben:zstd-jni:1.5.5-5 com.github.spotbugs:spotbugs-annotations:3.1.9 com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.21.1 +com.google.errorprone:error_prone_annotations:2.22.0 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:32.1.2-jre com.google.inject:guice:6.0.0 @@ -187,7 +187,7 @@ org.slf4j:slf4j-api:1.7.36 org.slf4j:slf4j-jdk14:1.7.36 org.slf4j:slf4j-simple:1.7.36 org.tukaani:xz:1.9 -org.xerial.snappy:snappy-java:1.1.10.3 +org.xerial.snappy:snappy-java:1.1.10.4 software.amazon.ion:ion-java:1.0.2 xerces:xercesImpl:2.12.2 diff --git a/vespalib/src/vespa/vespalib/text/utf8.h b/vespalib/src/vespa/vespalib/text/utf8.h index 99e3f8cfe13..489b16b1ed4 100644 --- a/vespalib/src/vespa/vespalib/text/utf8.h +++ b/vespalib/src/vespa/vespalib/text/utf8.h @@ -321,6 +321,7 @@ public: return i; } + const char* get_current_ptr() const noexcept { return _p; } }; |